forked from Blendman974/faceclaw
Merge branch 'main' into ring-health-fixes
This commit is contained in:
commit
edef7138fd
29 files changed
+1089
-234
No files matched your search
+120
-22
@@ -13,6 +13,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -61,12 +62,43 @@ public class FaceclawVoiceController {
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
};
|
||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||
//
|
||||
// On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0)
|
||||
// runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of
|
||||
// zero tail padding, capped at 30s. Re-decoding on Moonshine's
|
||||
// TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that
|
||||
// encode, growing with the utterance, roughly 1.4x/second while the user is
|
||||
// still speaking, so onboardModelKind gates that loop off for Whisper --
|
||||
// see processRecognizer(). Whisper is also known to hallucinate text on
|
||||
// near-silent input; recognizeTranscriptSegment() gates that too.
|
||||
private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8";
|
||||
private static final String[] ASR_WHISPER_MODEL_FILES = {
|
||||
"base.en-encoder.int8.onnx",
|
||||
"base.en-decoder.int8.onnx",
|
||||
"base.en-tokens.txt"
|
||||
};
|
||||
// Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f
|
||||
// buffer) a segment is treated as silence and never reaches the Whisper
|
||||
// recognizer at all, rather than risking a hallucinated non-answer. Picked
|
||||
// conservatively low (well under typical mic noise floor already seen in
|
||||
// this pipeline's normalization target) -- UNTESTED on real hardware, tune
|
||||
// against real glasses captures rather than trusting this number.
|
||||
private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f;
|
||||
|
||||
private enum VoiceInputMode {
|
||||
ONBOARD, // on-phone Moonshine transcription
|
||||
ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind)
|
||||
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
/** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */
|
||||
private enum OnboardModelKind {
|
||||
MOONSHINE,
|
||||
WHISPER
|
||||
}
|
||||
|
||||
private final Context appContext;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
@@ -86,6 +118,7 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean usePhoneMic;
|
||||
private boolean activePhoneMic;
|
||||
private VoiceInputMode mode = VoiceInputMode.CLOUD;
|
||||
private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE;
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
|
||||
@@ -150,6 +183,17 @@ public class FaceclawVoiceController {
|
||||
this.saveRecordings = saveRecordings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which on-device model {@link #start}("onboard") should load: "whisper"
|
||||
* selects the second on-device model (sherpa-onnx offline Whisper); any
|
||||
* other value (including null/absent) keeps the existing Moonshine model,
|
||||
* so callers that never call this see unchanged behavior. Must be set
|
||||
* before {@link #start}; has no effect in CLOUD mode.
|
||||
*/
|
||||
public void setOnboardModelKind(String kind) {
|
||||
this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE;
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
@@ -222,20 +266,26 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
start(requestedMode, 0);
|
||||
}
|
||||
|
||||
public void start(String requestedMode, int captureId) {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
|
||||
emitStatus("Voice control needs an active G2 connection.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
mode = parseMode(requestedMode);
|
||||
activePhoneMic = usePhoneMic;
|
||||
started = true;
|
||||
audioStarted = false;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
@@ -306,18 +356,19 @@ public class FaceclawVoiceController {
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
private void runLoop(int captureId) {
|
||||
try {
|
||||
deleteLegacyKwsFiles();
|
||||
VoiceInputMode currentMode = mode;
|
||||
OnboardModelKind currentOnboardKind = onboardModelKind;
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
File modelDir = findAsrModelDir();
|
||||
File modelDir = findAsrModelDir(currentOnboardKind);
|
||||
if (modelDir == null) {
|
||||
emitStatus("Voice model not downloaded (see Settings > Voice).");
|
||||
return;
|
||||
}
|
||||
emitStatus("Loading transcription model...");
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind));
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
}
|
||||
@@ -387,6 +438,7 @@ public class FaceclawVoiceController {
|
||||
audioStarted = false;
|
||||
workerThread = null;
|
||||
}
|
||||
emitStopped(captureId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,32 +505,48 @@ public class FaceclawVoiceController {
|
||||
return b.array();
|
||||
}
|
||||
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) {
|
||||
OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder()
|
||||
.setNumThreads(1);
|
||||
if (kind == OnboardModelKind.WHISPER) {
|
||||
modelConfig
|
||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath())
|
||||
.setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath())
|
||||
.setLanguage("en")
|
||||
.setTask("transcribe")
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath());
|
||||
} else {
|
||||
modelConfig
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath());
|
||||
}
|
||||
return OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setModelConfig(modelConfig.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* The Moonshine model directory, populated by the download flow in
|
||||
* asr-model.ts (releases before 0.5.0 copied the same files out of the
|
||||
* APK, so upgraded installs are already complete). Null when any file is
|
||||
* The on-device model directory for the given kind, populated by the
|
||||
* download flow in asr-model.ts (releases before 0.5.0 copied the
|
||||
* Moonshine files out of the APK directly, so upgraded installs are
|
||||
* already complete for that model). Null when any expected file is
|
||||
* missing, i.e. the model still needs to be downloaded.
|
||||
*/
|
||||
private File findAsrModelDir() {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
|
||||
for (String fileName : ASR_MODEL_FILES) {
|
||||
private File findAsrModelDir(OnboardModelKind kind) {
|
||||
boolean whisper = kind == OnboardModelKind.WHISPER;
|
||||
String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR;
|
||||
String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES;
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName);
|
||||
for (String fileName : fileNames) {
|
||||
File file = new File(modelDir, fileName);
|
||||
if (!file.exists() || file.length() == 0) {
|
||||
return null;
|
||||
@@ -658,6 +726,16 @@ public class FaceclawVoiceController {
|
||||
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
// Each Whisper call re-encodes the whole buffer plus ~10s of tail
|
||||
// padding (see the ASR_WHISPER_* comment above),
|
||||
// so it skips the live-partial redecode Moonshine does on this
|
||||
// interval and only decodes when a segment commits (8s buffer fill)
|
||||
// or the utterance ends (decodeTranscript(true) in runLoop()). This
|
||||
// means no live preview text while speaking in Whisper mode -- status
|
||||
// stays "Listening..." until release. This is a deliberate tradeoff.
|
||||
if (onboardModelKind == OnboardModelKind.WHISPER) {
|
||||
return;
|
||||
}
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES
|
||||
&& now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
|
||||
@@ -767,6 +845,14 @@ public class FaceclawVoiceController {
|
||||
return "";
|
||||
}
|
||||
float[] segment = Arrays.copyOf(transcriptSamples, sampleCount);
|
||||
// Whisper hallucinates text on near-silent input (a known quirk of the
|
||||
// model, not this pipeline); gate it on the PRE-normalization peak, since
|
||||
// normalizePeak() below would otherwise amplify true silence right up to
|
||||
// the target level and hide the very thing being checked for. Moonshine
|
||||
// does not share this failure mode in practice, so it is left unchanged.
|
||||
if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) {
|
||||
return "";
|
||||
}
|
||||
normalizePeak(segment);
|
||||
OfflineStream offlineStream = currentRecognizer.createStream();
|
||||
try {
|
||||
@@ -780,7 +866,7 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
}
|
||||
|
||||
private static void normalizePeak(float[] samples) {
|
||||
private static float peakAmplitude(float[] samples) {
|
||||
float peak = 0f;
|
||||
for (float s : samples) {
|
||||
float a = Math.abs(s);
|
||||
@@ -788,6 +874,11 @@ public class FaceclawVoiceController {
|
||||
peak = a;
|
||||
}
|
||||
}
|
||||
return peak;
|
||||
}
|
||||
|
||||
private static void normalizePeak(float[] samples) {
|
||||
float peak = peakAmplitude(samples);
|
||||
if (peak <= 0f) {
|
||||
return;
|
||||
}
|
||||
@@ -805,7 +896,7 @@ public class FaceclawVoiceController {
|
||||
(committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE;
|
||||
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
|
||||
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
|
||||
Log.i(TAG, "Moonshine decode final=" + isFinal
|
||||
Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal
|
||||
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec)
|
||||
+ " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE)
|
||||
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
|
||||
@@ -930,6 +1021,13 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(currentListener::onSpeechEnd);
|
||||
}
|
||||
|
||||
private void emitStopped(int captureId) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener != null) {
|
||||
mainHandler.post(() -> currentListener.onStopped(captureId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||
* stop without a button release.
|
||||
|
||||
+3
@@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener {
|
||||
*/
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/** Capture has finished, after the final transcript or a startup/recognition error. */
|
||||
void onStopped(int captureId);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
|
||||
@@ -162,7 +162,7 @@ public class OfflineModelConfig {
|
||||
public static class Builder {
|
||||
private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig();
|
||||
private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig();
|
||||
private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig();
|
||||
private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build();
|
||||
private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig();
|
||||
private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build();
|
||||
private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig();
|
||||
@@ -191,6 +191,11 @@ public class OfflineModelConfig {
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setWhisper(OfflineWhisperModelConfig whisper) {
|
||||
this.whisper = whisper;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTokens(String tokens) {
|
||||
this.tokens = tokens;
|
||||
return this;
|
||||
@@ -228,16 +233,6 @@ class OfflineParaformerModelConfig {
|
||||
private final QnnConfig qnnConfig = new QnnConfig();
|
||||
}
|
||||
|
||||
class OfflineWhisperModelConfig {
|
||||
private final String encoder = "";
|
||||
private final String decoder = "";
|
||||
private final String language = "en";
|
||||
private final String task = "transcribe";
|
||||
private final int tailPaddings = 1000;
|
||||
private final boolean enableTokenTimestamps = false;
|
||||
private final boolean enableSegmentTimestamps = false;
|
||||
}
|
||||
|
||||
class OfflineFireRedAsrModelConfig {
|
||||
private final String encoder = "";
|
||||
private final String decoder = "";
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
/**
|
||||
* Config for sherpa-onnx's offline (non-streaming) Whisper backend.
|
||||
*
|
||||
* Field names and types here are load-bearing: the native JNI glue reads
|
||||
* them by name via reflection (GetFieldID), not through the getters below,
|
||||
* so they must match sherpa-onnx's own OfflineWhisperModelConfig exactly
|
||||
* (verified against sherpa-onnx/jni/offline-recognizer.cc at the pinned
|
||||
* release tag: it unconditionally reads encoder, decoder, language, task,
|
||||
* tailPaddings, enableTokenTimestamps and enableSegmentTimestamps off the
|
||||
* OfflineModelConfig's "whisper" field). This mirrors the
|
||||
* OfflineMoonshineModelConfig.java pattern already used in this package.
|
||||
*/
|
||||
public class OfflineWhisperModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
private final String language;
|
||||
private final String task;
|
||||
private final int tailPaddings;
|
||||
private final boolean enableTokenTimestamps;
|
||||
private final boolean enableSegmentTimestamps;
|
||||
|
||||
private OfflineWhisperModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
this.language = builder.language;
|
||||
this.task = builder.task;
|
||||
this.tailPaddings = builder.tailPaddings;
|
||||
this.enableTokenTimestamps = builder.enableTokenTimestamps;
|
||||
this.enableSegmentTimestamps = builder.enableSegmentTimestamps;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public String getLanguage() {
|
||||
return language;
|
||||
}
|
||||
|
||||
public String getTask() {
|
||||
return task;
|
||||
}
|
||||
|
||||
public int getTailPaddings() {
|
||||
return tailPaddings;
|
||||
}
|
||||
|
||||
public boolean getEnableTokenTimestamps() {
|
||||
return enableTokenTimestamps;
|
||||
}
|
||||
|
||||
public boolean getEnableSegmentTimestamps() {
|
||||
return enableSegmentTimestamps;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
private String language = "en";
|
||||
private String task = "transcribe";
|
||||
private int tailPaddings = 1000;
|
||||
private boolean enableTokenTimestamps = false;
|
||||
private boolean enableSegmentTimestamps = false;
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLanguage(String language) {
|
||||
this.language = language;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTask(String task) {
|
||||
this.task = task;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTailPaddings(int tailPaddings) {
|
||||
this.tailPaddings = tailPaddings;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEnableTokenTimestamps(boolean enableTokenTimestamps) {
|
||||
this.enableTokenTimestamps = enableTokenTimestamps;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEnableSegmentTimestamps(boolean enableSegmentTimestamps) {
|
||||
this.enableSegmentTimestamps = enableSegmentTimestamps;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OfflineWhisperModelConfig build() {
|
||||
return new OfflineWhisperModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,9 @@
|
||||
* Add Wear OS watch battery indicator to the top bar and Glanceboard
|
||||
* Glanceboard tap setting includes a Disable option
|
||||
* Refinements to builtin games
|
||||
* Optimize bluetooth message transport
|
||||
* Short-the-long gesture from screen off activates the glanceboard
|
||||
* Glanceboard has a 2x3 layout option
|
||||
|
||||
## 0.7.0 - 2026-09-12
|
||||
|
||||
|
||||
@@ -89,11 +89,22 @@ export class VoiceDraft {
|
||||
}
|
||||
|
||||
private stopAndFinalize(): void {
|
||||
// Install fallback first: stop may synchronously deliver the final.
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
const generation = this.generation;
|
||||
const afterStop = () => {
|
||||
if (generation !== this.generation || !this.isFinishing()) return;
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
};
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
|
||||
// Native recognition can take longer than the synchronous stop join.
|
||||
// Its final callback may finish the draft before this promise resolves.
|
||||
const stopped = this.actions.stopVoiceCapture();
|
||||
if (!stopped) { afterStop(); return; }
|
||||
void stopped.then(afterStop, (error) => {
|
||||
if (generation === this.generation) this.cancel(String(error));
|
||||
});
|
||||
} else {
|
||||
afterStop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { onAnySettingChanged } from "../../ui/dashboard-settings";
|
||||
import { glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings";
|
||||
import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout";
|
||||
import { glanceLayout, glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings";
|
||||
import { resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout";
|
||||
import { type GlanceWidget } from "./widget";
|
||||
import { findGlanceWidget, glanceWidgetSpans } from "./widgets";
|
||||
|
||||
@@ -27,11 +27,18 @@ export class GlanceBoard {
|
||||
private regions: LiveRegion[] = [];
|
||||
private started = false;
|
||||
private unsubscribeSettings: (() => void) | null = null;
|
||||
private currentLayout: GlanceLayout;
|
||||
|
||||
constructor(
|
||||
private readonly requestRender: () => void,
|
||||
readonly layout: GlanceLayout = QUADRANT_LAYOUT,
|
||||
) {}
|
||||
private readonly fixedLayout?: GlanceLayout,
|
||||
) {
|
||||
this.currentLayout = fixedLayout ?? glanceLayout();
|
||||
}
|
||||
|
||||
get layout(): GlanceLayout {
|
||||
return this.currentLayout;
|
||||
}
|
||||
|
||||
get width(): number {
|
||||
return this.layout.width;
|
||||
@@ -65,9 +72,12 @@ export class GlanceBoard {
|
||||
/** Rebuild the live regions from the settings, keeping widgets whose region is unchanged. */
|
||||
private applySlotSettings(): void {
|
||||
if (!this.started) return;
|
||||
const layout = this.fixedLayout ?? glanceLayout();
|
||||
const layoutChanged = layout !== this.currentLayout;
|
||||
this.currentLayout = layout;
|
||||
const previous = new Map(this.regions.map((live) => [live.key, live]));
|
||||
const next: LiveRegion[] = [];
|
||||
let changed = false;
|
||||
let changed = layoutChanged;
|
||||
for (const region of this.currentRegions()) {
|
||||
const key = regionKey(region);
|
||||
const kept = previous.get(key);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { drawSelectionHighlight, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu";
|
||||
@@ -12,6 +12,8 @@ import { GlanceBoard } from "./board";
|
||||
import {
|
||||
clearConflictingSlots,
|
||||
glanceboardEnabledSetting,
|
||||
glanceLayout,
|
||||
glanceLayoutSetting,
|
||||
glanceShowLinesSetting,
|
||||
glanceShowOnHeadTiltSetting,
|
||||
glanceShowOnLongPressSetting,
|
||||
@@ -20,7 +22,7 @@ import {
|
||||
glanceTapDurationSetting,
|
||||
type GlanceSlotChoice,
|
||||
} from "./glanceboard-settings";
|
||||
import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers } from "./layout";
|
||||
import { resolveGlanceRegions, slotDividers } from "./layout";
|
||||
import { glanceWidgetSpans } from "./widgets";
|
||||
|
||||
export const GLANCEBOARD_WINDOW_ID = "glanceboard";
|
||||
@@ -49,24 +51,33 @@ function pageMenuLayout(width: number) {
|
||||
* and the menu (enable toggle, live preview, slot contents, settings). Right
|
||||
* half: a scaled preview of the board's slots with their dividers and the
|
||||
* name of the widget in each. "Select Contents" moves focus into that
|
||||
* preview, where scroll picks a quadrant and a tap opens the picker for it;
|
||||
* preview, where scroll picks a slot and a tap opens the picker for it;
|
||||
* a double-tap returns focus to the menu.
|
||||
*/
|
||||
class GlanceboardHomeLayer implements Layer {
|
||||
private menu: MenuLayer | null = null;
|
||||
private menuTop = 0;
|
||||
private menuHeight = 0;
|
||||
private focus: "menu" | "preview" = "menu";
|
||||
private selectedSlot = 0;
|
||||
|
||||
constructor(private readonly setPreviewVisible: (visible: boolean) => void) {}
|
||||
|
||||
private currentMenu(ctx: LayerContext, menuTop: number): MenuLayer {
|
||||
if (!this.menu || this.menuTop !== menuTop) {
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
if (!this.menu || this.menuTop !== menuTop || this.menuHeight !== height) {
|
||||
const half = (width / 2) | 0;
|
||||
const items: MenuItem[] = [
|
||||
toggleSettingMenuItem(glanceboardEnabledSetting),
|
||||
{
|
||||
label: "Preview",
|
||||
onSelect: (menuCtx) => menuCtx.stack.push(new GlancePreviewLayer(() => menuCtx.actions.requestRender())),
|
||||
onSelect: (menuCtx) => {
|
||||
menuCtx.stack.push(new GlancePreviewLayer(
|
||||
() => menuCtx.actions.requestRender(),
|
||||
() => this.setPreviewVisible(false),
|
||||
));
|
||||
this.setPreviewVisible(true);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Select Contents",
|
||||
@@ -80,6 +91,7 @@ class GlanceboardHomeLayer implements Layer {
|
||||
},
|
||||
];
|
||||
this.menuTop = menuTop;
|
||||
this.menuHeight = height;
|
||||
this.menu = new MenuLayer(null, items, {
|
||||
x: 8, y: menuTop, width: half - 16,
|
||||
showBorder: false, minHeight: 0, maxHeight: height - menuTop,
|
||||
@@ -92,9 +104,10 @@ class GlanceboardHomeLayer implements Layer {
|
||||
private previewRect(width: number, height: number): { x: number; y: number; width: number; height: number } {
|
||||
const half = (width / 2) | 0;
|
||||
const availableWidth = width - half - 2 * PREVIEW_MARGIN;
|
||||
const scale = Math.min(availableWidth / QUADRANT_LAYOUT.width, (height - 2 * PREVIEW_MARGIN) / QUADRANT_LAYOUT.height);
|
||||
const previewWidth = Math.round(QUADRANT_LAYOUT.width * scale);
|
||||
const previewHeight = Math.round(QUADRANT_LAYOUT.height * scale);
|
||||
const layout = glanceLayout();
|
||||
const scale = Math.min(availableWidth / layout.width, (height - 2 * PREVIEW_MARGIN) / layout.height);
|
||||
const previewWidth = Math.round(layout.width * scale);
|
||||
const previewHeight = Math.round(layout.height * scale);
|
||||
return {
|
||||
x: half + PREVIEW_MARGIN + (((availableWidth - previewWidth) / 2) | 0),
|
||||
y: ((height - previewHeight) / 2) | 0,
|
||||
@@ -123,6 +136,7 @@ class GlanceboardHomeLayer implements Layer {
|
||||
const menuCtx: LayerContext = { ...ctx, stack: Object.assign(Object.create(ctx.stack), { isFocused: () => menuFocused }) };
|
||||
menu.paint(menuCtx, () => image);
|
||||
|
||||
this.selectedSlot = Math.min(this.selectedSlot, glanceLayout().slots.length - 1);
|
||||
const preview = this.previewRect(width, height);
|
||||
drawSlotPreview(image, preview, {
|
||||
selectedSlot: this.selectedSlot,
|
||||
@@ -133,7 +147,8 @@ class GlanceboardHomeLayer implements Layer {
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
if (this.focus === "preview") {
|
||||
const slotCount = QUADRANT_LAYOUT.slots.length;
|
||||
const slotCount = glanceLayout().slots.length;
|
||||
this.selectedSlot = Math.min(this.selectedSlot, slotCount - 1);
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.selectedSlot = (this.selectedSlot + slotCount - 1) % slotCount;
|
||||
@@ -166,7 +181,7 @@ class GlanceboardHomeLayer implements Layer {
|
||||
* The board's slots scaled into `rect`: dividers (full brightness, or the
|
||||
* dimmest dotted line when "Show lines" is off), the widget name centred in
|
||||
* each region (a merged double-height region has one name and no line
|
||||
* across it), and the selected quadrant highlighted.
|
||||
* across it), and the selected slot highlighted.
|
||||
*/
|
||||
function drawSlotPreview(
|
||||
image: GrayImage,
|
||||
@@ -174,7 +189,7 @@ function drawSlotPreview(
|
||||
options: { selectedSlot: number; selectionFocused: boolean },
|
||||
): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const layout = QUADRANT_LAYOUT;
|
||||
const layout = glanceLayout();
|
||||
const settings = glanceSlotSettings(layout);
|
||||
const choices = settings.map((setting) => setting.get());
|
||||
const regions = resolveGlanceRegions(layout, choices, glanceWidgetSpans);
|
||||
@@ -232,14 +247,15 @@ function drawDottedLine(image: GrayImage, x0: number, y0: number, x1: number, y1
|
||||
|
||||
/** The picker for one slot: every widget, the current one marked; choosing applies the one-per-board rule. */
|
||||
function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
|
||||
const setting = glanceSlotSettings(QUADRANT_LAYOUT)[slotIndex];
|
||||
const layout = glanceLayout();
|
||||
const setting = glanceSlotSettings(layout)[slotIndex];
|
||||
if (!setting) return;
|
||||
const current = setting.get();
|
||||
const items: MenuItem[] = setting.values.map((value) => ({
|
||||
label: setting.displayValue(value),
|
||||
onSelect: (menuCtx) => {
|
||||
setting.set(value);
|
||||
clearConflictingSlots(QUADRANT_LAYOUT, slotIndex, value);
|
||||
clearConflictingSlots(layout, slotIndex, value);
|
||||
menuCtx.stack.pop();
|
||||
},
|
||||
render: ({ image, x, y }) => {
|
||||
@@ -249,7 +265,7 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
|
||||
openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current)));
|
||||
}
|
||||
|
||||
/** Settings: which sleep gestures show the board, for how long, and the slot lines. */
|
||||
/** Settings: layout, sleep gestures, duration, and slot lines. */
|
||||
class GlanceSettingsLayer implements Layer {
|
||||
private menu: MenuLayer | null = null;
|
||||
|
||||
@@ -257,6 +273,7 @@ class GlanceSettingsLayer implements Layer {
|
||||
if (!this.menu) {
|
||||
const { width } = ctx.stack.getBaseSize();
|
||||
const items: MenuItem[] = [
|
||||
enumSettingMenuItem(glanceLayoutSetting),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
@@ -280,7 +297,7 @@ class GlanceSettingsLayer implements Layer {
|
||||
class GlancePreviewLayer implements Layer {
|
||||
private readonly board: GlanceBoard;
|
||||
|
||||
constructor(requestRender: () => void) {
|
||||
constructor(requestRender: () => void, private readonly onClose: () => void) {
|
||||
this.board = new GlanceBoard(requestRender);
|
||||
this.board.start();
|
||||
}
|
||||
@@ -302,27 +319,40 @@ class GlancePreviewLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.board.stop();
|
||||
this.onClose();
|
||||
}
|
||||
}
|
||||
|
||||
export function createGlanceboardAppWindow(options: InProcessAppOptions): InProcessWindow {
|
||||
return createInProcessWindow({
|
||||
let previewVisible = false;
|
||||
const heightMode = () => previewVisible && glanceLayoutSetting.get() === "2x3" ? "max" as const : "medium" as const;
|
||||
let unsubscribeSettings: (() => void) | undefined;
|
||||
const app = createInProcessWindow({
|
||||
appId: "glanceboard",
|
||||
windowId: GLANCEBOARD_WINDOW_ID,
|
||||
title: "Glanceboard",
|
||||
iconLetter: "Gb",
|
||||
icon: "eye",
|
||||
closeable: true,
|
||||
// 576x288: the board's own size, so the preview is pixel-exact.
|
||||
heightMode: "medium",
|
||||
// Settings and the contents picker stay in the default 576x288 viewport;
|
||||
// only the live six-slot preview needs the taller window.
|
||||
heightMode: heightMode(),
|
||||
actions: options.actions,
|
||||
// Not wrapped in YieldAtRootLayer: the home page routes double-click
|
||||
// itself (preview focus -> menu focus, menu focus -> sidebar).
|
||||
baseLayer: new GlanceboardHomeLayer(),
|
||||
baseLayer: new GlanceboardHomeLayer((visible) => {
|
||||
previewVisible = visible;
|
||||
app.setHeightMode(heightMode());
|
||||
}),
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
reconfigureSurface: options.reconfigureSurface,
|
||||
onClosed: options.onClosed,
|
||||
onClosed: () => {
|
||||
unsubscribeSettings?.();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
unsubscribeSettings = onAnySettingChanged(() => app.setHeightMode(heightMode()));
|
||||
return app;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
|
||||
import { QUADRANT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout";
|
||||
import { QUADRANT_LAYOUT, SIX_SLOT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout";
|
||||
import { type GlanceWidgetId } from "./widget";
|
||||
import { glanceWidgetSpans } from "./widgets";
|
||||
|
||||
@@ -28,6 +28,19 @@ export function glanceSlotChoiceLabel(choice: GlanceSlotChoice): string {
|
||||
*/
|
||||
const DEFAULT_QUADRANT_CHOICES: readonly GlanceSlotChoice[] = ["system-card", "calendar", "music", "calendar"];
|
||||
|
||||
export const glanceLayoutSetting = new ConfigSettingEnum<"2x2" | "2x3">({
|
||||
id: "glanceboard-layout",
|
||||
label: "Layout",
|
||||
storageKey: "glanceboard.layout",
|
||||
defaultValue: "2x2",
|
||||
values: ["2x2", "2x3"],
|
||||
description: "Choose two columns with two or three rows of widgets.",
|
||||
});
|
||||
|
||||
export function glanceLayout(): GlanceLayout {
|
||||
return glanceLayoutSetting.get() === "2x3" ? SIX_SLOT_LAYOUT : QUADRANT_LAYOUT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether sleep-time gestures show the board at all. Off restores the plain
|
||||
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
|
||||
@@ -79,7 +92,7 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
|
||||
label: "Show on long press",
|
||||
storageKey: "glanceboard.showOnLongPress",
|
||||
defaultValue: true,
|
||||
description: "Holding a long-press while asleep shows the Glanceboard until the press is released.",
|
||||
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
|
||||
});
|
||||
|
||||
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
|
||||
@@ -101,17 +114,19 @@ export const glanceShowLinesSetting = new ConfigSettingBoolean({
|
||||
});
|
||||
|
||||
/**
|
||||
* One picker per layout slot, keyed by layout id and slot index so a future
|
||||
* layout gets its own stored choices rather than inheriting the quadrants'.
|
||||
* Both grid sizes share the original storage keys so changing row count
|
||||
* preserves the first four choices and remembers the hidden bottom row.
|
||||
*/
|
||||
export function glanceSlotSettings(layout: GlanceLayout = QUADRANT_LAYOUT): ConfigSettingEnum<GlanceSlotChoice>[] {
|
||||
export function glanceSlotSettings(layout: GlanceLayout = glanceLayout()): ConfigSettingEnum<GlanceSlotChoice>[] {
|
||||
const grid = layout === QUADRANT_LAYOUT || layout === SIX_SLOT_LAYOUT;
|
||||
const storageId = grid ? QUADRANT_LAYOUT.id : layout.id;
|
||||
return layout.slots.map(
|
||||
(slot, index) =>
|
||||
new ConfigSettingEnum<GlanceSlotChoice>({
|
||||
id: `glanceboard-${layout.id}-slot-${index}`,
|
||||
label: slot.label,
|
||||
storageKey: `glanceboard.${layout.id}.slot.${index}`,
|
||||
defaultValue: (layout === QUADRANT_LAYOUT ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none",
|
||||
storageKey: `glanceboard.${storageId}.slot.${index}`,
|
||||
defaultValue: (grid ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none",
|
||||
values: SLOT_CHOICES,
|
||||
formatValue: glanceSlotChoiceLabel,
|
||||
description: `Which widget fills the ${slot.label.toLowerCase()} slot of the Glanceboard.`,
|
||||
@@ -130,9 +145,13 @@ export function clearConflictingSlots(layout: GlanceLayout, changedIndex: number
|
||||
const settings = glanceSlotSettings(layout);
|
||||
const spans = glanceWidgetSpans(choice);
|
||||
const cleared: number[] = [];
|
||||
let keptPartner = false;
|
||||
settings.forEach((setting, index) => {
|
||||
if (index === changedIndex || setting.get() !== choice) return;
|
||||
if (spans && slotsVerticallyAdjacent(layout, changedIndex, index)) return;
|
||||
if (spans && !keptPartner && slotsVerticallyAdjacent(layout, changedIndex, index)) {
|
||||
keptPartner = true;
|
||||
return;
|
||||
}
|
||||
setting.set("none");
|
||||
cleared.push(index);
|
||||
});
|
||||
|
||||
@@ -3,12 +3,12 @@ import { GlanceBoard } from "./board";
|
||||
import { createGlanceboardAppWindow, GLANCEBOARD_SURFACE_ID, GLANCEBOARD_WINDOW_ID } from "./glanceboard-app";
|
||||
import {
|
||||
glanceboardEnabledSetting,
|
||||
glanceLayout,
|
||||
glanceShowOnHeadTiltSetting,
|
||||
glanceShowOnLongPressSetting,
|
||||
glanceShowOnTap,
|
||||
glanceTapTimeoutMs,
|
||||
} from "./glanceboard-settings";
|
||||
import { QUADRANT_LAYOUT } from "./layout";
|
||||
|
||||
const glanceboardApp: AppDefinition = {
|
||||
appId: "glanceboard",
|
||||
@@ -16,13 +16,16 @@ const glanceboardApp: AppDefinition = {
|
||||
icon: "eye",
|
||||
launch: (ctx) => ctx.launchInProcessApp(GLANCEBOARD_WINDOW_ID, GLANCEBOARD_SURFACE_ID, createGlanceboardAppWindow),
|
||||
glanceboard: {
|
||||
size: { width: QUADRANT_LAYOUT.width, height: QUADRANT_LAYOUT.height },
|
||||
get size() {
|
||||
const layout = glanceLayout();
|
||||
return { width: layout.width, height: layout.height };
|
||||
},
|
||||
isEnabled: () => glanceboardEnabledSetting.get(),
|
||||
showOnTap: glanceShowOnTap,
|
||||
tapTimeoutMs: glanceTapTimeoutMs,
|
||||
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
|
||||
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
|
||||
createBoard: (requestRender) => new GlanceBoard(requestRender, QUADRANT_LAYOUT),
|
||||
createBoard: (requestRender) => new GlanceBoard(requestRender),
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -29,6 +29,20 @@ export const QUADRANT_LAYOUT: GlanceLayout = {
|
||||
],
|
||||
};
|
||||
|
||||
/** Six 288x144 slots: the same first two rows, plus a third at the bottom. */
|
||||
export const SIX_SLOT_LAYOUT: GlanceLayout = {
|
||||
id: "six-slots",
|
||||
width: GLANCEBOARD_WIDTH,
|
||||
height: 432,
|
||||
slots: [
|
||||
...QUADRANT_LAYOUT.slots.slice(0, 2),
|
||||
{ label: "Middle left", rect: { x: 0, y: 144, width: 288, height: 144 } },
|
||||
{ label: "Middle right", rect: { x: 288, y: 144, width: 288, height: 144 } },
|
||||
{ label: "Bottom left", rect: { x: 0, y: 288, width: 288, height: 144 } },
|
||||
{ label: "Bottom right", rect: { x: 288, y: 288, width: 288, height: 144 } },
|
||||
],
|
||||
};
|
||||
|
||||
/** Whether two slots stack vertically with matching width (one region if merged). */
|
||||
export function slotsVerticallyAdjacent(layout: GlanceLayout, a: number, b: number): boolean {
|
||||
const first = layout.slots[a]?.rect;
|
||||
|
||||
@@ -730,7 +730,10 @@ class MicSession {
|
||||
},
|
||||
});
|
||||
engine.setListener(this.captionListenerProxy);
|
||||
engine.setAsrModelDir(isAsrModelReady() ? moonshineModelDir() : null);
|
||||
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
|
||||
// that only ever uses Moonshine; unaffected by the two-model on-device
|
||||
// Whisper addition in native/voice-control.ts's push-to-talk path.
|
||||
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
|
||||
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
|
||||
engine.start();
|
||||
this.captionEngine = engine;
|
||||
|
||||
@@ -328,17 +328,20 @@ function voiceSpeakersMenu(): MenuLayer {
|
||||
() => startMicModelDownload("speaker-embedding"),
|
||||
),
|
||||
modelItem(
|
||||
// Live captions always use Moonshine (see the matching note in
|
||||
// mic-session.ts); the second on-device model (Whisper) is a
|
||||
// push-to-talk dictation option only, managed in Settings > Voice.
|
||||
"Caption ASR model",
|
||||
isAsrModelReady,
|
||||
() => isAsrModelReady("moonshine"),
|
||||
() => {
|
||||
const state = asrModelState();
|
||||
const state = asrModelState("moonshine");
|
||||
if (state.status === "ready") return "ready";
|
||||
if (state.status === "downloading") {
|
||||
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
|
||||
}
|
||||
return "download (141 MB)";
|
||||
},
|
||||
startAsrModelDownload,
|
||||
() => startAsrModelDownload("moonshine"),
|
||||
),
|
||||
modelItem(
|
||||
"Re-diarization model",
|
||||
|
||||
@@ -1912,9 +1912,9 @@ class DashboardController {
|
||||
return this.beginVoiceCapture("ptt", endpointing);
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
private stopVoiceCapture(): Promise<void> | void {
|
||||
++this.pttCaptureGeneration;
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
return voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private startContinuousVoiceCapture(): void {
|
||||
|
||||
@@ -77,7 +77,7 @@ export class GlanceHost {
|
||||
const provider = this.options.getProvider();
|
||||
if (!provider?.isEnabled()) return null;
|
||||
if (gesture === "click" && !provider.showOnTap()) return null;
|
||||
if (gesture === "long-press" && !provider.showOnLongPress()) return null;
|
||||
if ((gesture === "long-press" || gesture === "short-then-long-press") && !provider.showOnLongPress()) return null;
|
||||
if (gesture === "head-tilt" && !provider.showOnHeadTilt()) return null;
|
||||
return glanceEventForGesture(gesture, this.isVisible());
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* The Glanceboard's visibility rules, as a pure reducer so they can be tested
|
||||
* without the display plumbing. The board is an alternate sleep-time display:
|
||||
* while the shell is asleep a press (single tap, or a head-tilt wake) shows it
|
||||
* for a short while, a hold (long-press) shows it until the press is released,
|
||||
* for a short while, a hold (long-press or short-then-long-press) shows it until release,
|
||||
* and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds
|
||||
* these events in and applies the resulting visibility to the compositor.
|
||||
*
|
||||
@@ -49,8 +49,7 @@ export function reduceGlance(
|
||||
case "hold":
|
||||
return { visible: true, holding: true, hideAtMs: null };
|
||||
case "release":
|
||||
// Releases also follow presses the board was never holding for (a
|
||||
// tap-then-hold, a release delivered late); those mean nothing here.
|
||||
// Releases delivered late or after a disabled hold gesture mean nothing here.
|
||||
return state.holding ? GLANCE_HIDDEN : state;
|
||||
case "timeout":
|
||||
if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state;
|
||||
@@ -67,7 +66,7 @@ export function reduceGlance(
|
||||
* already handles or ignores while asleep.
|
||||
*/
|
||||
export function glanceEventForGesture(
|
||||
gesture: "click" | "double-click" | "long-press" | "long-press-release" | "head-tilt" | string,
|
||||
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | string,
|
||||
boardVisible: boolean,
|
||||
): GlanceEvent | null {
|
||||
switch (gesture) {
|
||||
@@ -75,6 +74,7 @@ export function glanceEventForGesture(
|
||||
case "head-tilt":
|
||||
return { type: "press" };
|
||||
case "long-press":
|
||||
case "short-then-long-press":
|
||||
return { type: "hold" };
|
||||
case "long-press-release":
|
||||
return { type: "release" };
|
||||
|
||||
+172
-95
@@ -4,50 +4,107 @@ declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Download management for the on-device transcription model (Moonshine, via
|
||||
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
|
||||
* demand into the same filesDir location that earlier releases copied the
|
||||
* bundled files to, so upgraded installs that already used on-device
|
||||
* transcription need no download. FaceclawVoiceController reads the files
|
||||
* from this directory.
|
||||
* Download management for the on-device transcription models (sherpa-onnx
|
||||
* offline recognizers). Models are fetched on demand into filesDir rather
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
|
||||
* three files rather than one; they download sequentially through
|
||||
* Two models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
* upgraded installs that already used on-device transcription need no
|
||||
* download.
|
||||
* - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en,
|
||||
* int8-quantized. Picked over tiny.en for materially better accuracy
|
||||
* (Whisper's own tiny/base WER gap is real and well documented) on the
|
||||
* reasoning that FaceclawVoiceController.java no longer re-decodes a
|
||||
* Whisper segment on every live-partial tick — see its ASR_WHISPER_*
|
||||
* comment — so base.en's extra compute is a one-time cost per utterance,
|
||||
* not a cost paid ~1.4x/second while the user is speaking. tiny.en is an
|
||||
* easy swap (this file's numbers only) if base.en proves too slow on a
|
||||
* real phone; nobody has measured that yet. Files are csukuangfj's own
|
||||
* (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx
|
||||
* release asset, matching the mirror-not-the-GH-tarball approach already
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODEL = {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
] as AsrModelFile[],
|
||||
totalBytes: 141300566,
|
||||
type AsrModelDef = {
|
||||
label: string;
|
||||
dirName: string;
|
||||
baseUrl: string;
|
||||
files: AsrModelFile[];
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
moonshine: {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
],
|
||||
totalBytes: 141300566,
|
||||
},
|
||||
"whisper-base-en": {
|
||||
label: "Whisper (English, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-en-int8",
|
||||
// csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx
|
||||
// maintainer's own mirror of the project's whisper export, individual
|
||||
// files (no tar.bz2-unpack problem to begin with, but kept as the same
|
||||
// per-file-with-pinned-hash shape as Moonshine above for consistency).
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "base.en-encoder.int8.onnx",
|
||||
sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base.en-decoder.int8.onnx",
|
||||
sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41",
|
||||
sizeBytes: 130669978,
|
||||
},
|
||||
{
|
||||
name: "base.en-tokens.txt",
|
||||
sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
|
||||
sizeBytes: 835554,
|
||||
},
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -56,121 +113,141 @@ export type AsrModelState = {
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let downloader: any = null;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
let completedBytes = 0;
|
||||
let currentFileBytes = 0;
|
||||
const stateListeners = new Set<(state: AsrModelState) => void>();
|
||||
type ModelRuntime = {
|
||||
downloader: any;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
completedBytes: number;
|
||||
currentFileBytes: number;
|
||||
stateListeners: Set<(state: AsrModelState) => void>;
|
||||
};
|
||||
|
||||
function modelDirPath(): string {
|
||||
function freshRuntime(): ModelRuntime {
|
||||
return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() };
|
||||
}
|
||||
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`;
|
||||
}
|
||||
|
||||
function filePath(file: AsrModelFile): string {
|
||||
return `${modelDirPath()}/${file.name}`;
|
||||
function filePath(id: AsrModelId, file: AsrModelFile): string {
|
||||
return `${modelDirPath(id)}/${file.name}`;
|
||||
}
|
||||
|
||||
function isFilePresent(file: AsrModelFile): boolean {
|
||||
function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean {
|
||||
try {
|
||||
const javaFile = new java.io.File(filePath(file));
|
||||
const javaFile = new java.io.File(filePath(id, file));
|
||||
return javaFile.exists() && javaFile.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAsrModelReady(): boolean {
|
||||
export function isAsrModelReady(id: AsrModelId): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
return ASR_MODEL.files.every(isFilePresent);
|
||||
return ASR_MODELS[id].files.every((file) => isFilePresent(id, file));
|
||||
}
|
||||
|
||||
export function asrModelState(): AsrModelState {
|
||||
if (downloader) {
|
||||
export function asrModelState(id: AsrModelId): AsrModelState {
|
||||
const runtime = runtimes[id];
|
||||
const totalBytes = ASR_MODELS[id].totalBytes;
|
||||
if (runtime.downloader) {
|
||||
return {
|
||||
status: "downloading",
|
||||
bytesDownloaded: completedBytes + currentFileBytes,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: isAsrModelReady() ? "ready" : "absent",
|
||||
status: isAsrModelReady(id) ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void {
|
||||
const runtime = runtimes[id];
|
||||
runtime.stateListeners.add(listener);
|
||||
return () => runtime.stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = asrModelState();
|
||||
stateListeners.forEach((listener) => listener(state));
|
||||
function notifyStateChanged(id: AsrModelId): void {
|
||||
const state = asrModelState(id);
|
||||
runtimes[id].stateListeners.forEach((listener) => listener(state));
|
||||
}
|
||||
|
||||
export function startAsrModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isAsrModelReady()) return;
|
||||
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile();
|
||||
notifyStateChanged();
|
||||
export function startAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return;
|
||||
runtime.completedBytes = ASR_MODELS[id].files
|
||||
.filter((file) => isFilePresent(id, file))
|
||||
.reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
function downloadNextFile(): void {
|
||||
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
|
||||
function downloadNextFile(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
const def = ASR_MODELS[id];
|
||||
const nextFile = def.files.find((file) => !isFilePresent(id, file));
|
||||
if (!nextFile) {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
return;
|
||||
}
|
||||
currentFileBytes = 0;
|
||||
runtime.currentFileBytes = 0;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
currentFileBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
runtime.currentFileBytes = Number(bytes);
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
onDone: () => {
|
||||
completedBytes += nextFile.sizeBytes;
|
||||
currentFileBytes = 0;
|
||||
downloadNextFile();
|
||||
runtime.completedBytes += nextFile.sizeBytes;
|
||||
runtime.currentFileBytes = 0;
|
||||
downloadNextFile(id);
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`);
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${ASR_MODEL.baseUrl}${nextFile.name}`,
|
||||
filePath(nextFile),
|
||||
runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${def.baseUrl}${nextFile.name}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloader.start();
|
||||
runtime.downloader.start();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelAsrModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
export function cancelAsrModelDownload(id: AsrModelId): void {
|
||||
const runtime = runtimes[id];
|
||||
if (!runtime.downloader) return;
|
||||
runtime.downloader.cancel();
|
||||
runtime.downloader = null;
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
export function deleteAsrModel(): void {
|
||||
export function deleteAsrModel(id: AsrModelId): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelAsrModelDownload();
|
||||
cancelAsrModelDownload(id);
|
||||
try {
|
||||
for (const file of ASR_MODEL.files) {
|
||||
new java.io.File(filePath(file)).delete();
|
||||
new java.io.File(`${filePath(file)}.part`).delete();
|
||||
for (const file of ASR_MODELS[id].files) {
|
||||
new java.io.File(filePath(id, file)).delete();
|
||||
new java.io.File(`${filePath(id, file)}.part`).delete();
|
||||
}
|
||||
new java.io.File(modelDirPath()).delete();
|
||||
new java.io.File(modelDirPath(id)).delete();
|
||||
} catch (error) {
|
||||
console.error(`Voice model delete failed: ${String(error)}`);
|
||||
console.error(`Voice model delete failed (${id}): ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
@@ -26,7 +26,12 @@ export type VoiceControlState = {
|
||||
detail: string;
|
||||
};
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -103,6 +108,9 @@ export class FaceclawVoiceControlBridge {
|
||||
private listening = false;
|
||||
private detail = "";
|
||||
private started = false;
|
||||
private nativeCaptureFinished: Promise<void> = Promise.resolve();
|
||||
private nextNativeCaptureId = 1;
|
||||
private readonly nativeCaptureResolvers = new Map<number, () => void>();
|
||||
// The mic is a single shared stream; these are the reasons it is running.
|
||||
// The first holder starts capture (choosing the provider); the mic stops
|
||||
// when the last one releases. Transcript events broadcast to every
|
||||
@@ -184,8 +192,13 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
|
||||
stopPushToTalk(): void {
|
||||
stopPushToTalk(): Promise<void> | void {
|
||||
// Shared continuous capture keeps running; cloud providers finalize via
|
||||
// their own callbacks. Only a stopped native recognizer must be awaited.
|
||||
const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1
|
||||
? this.nativeCaptureFinished : undefined;
|
||||
this.releaseCapture("ptt", true);
|
||||
return finished;
|
||||
}
|
||||
|
||||
/** Begin continuous capture (Transcribe): the mic stays on until released. */
|
||||
@@ -314,7 +327,12 @@ export class FaceclawVoiceControlBridge {
|
||||
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
this.controller?.start("onboard");
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -323,7 +341,7 @@ export class FaceclawVoiceControlBridge {
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard") return null;
|
||||
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
@@ -483,6 +501,10 @@ export class FaceclawVoiceControlBridge {
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onStopped: (captureId: number) => {
|
||||
this.nativeCaptureResolvers.get(captureId)?.();
|
||||
this.nativeCaptureResolvers.delete(captureId);
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
|
||||
@@ -523,12 +523,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
"How R1 ring input reaches the phone. Only via glasses: the ring's own link to the glasses carries its gestures, and the phone never opens a Bluetooth connection to the ring. Direct: also connect to the ring from the phone (currently unreliable). Takes effect on the next connection to the glasses.",
|
||||
});
|
||||
|
||||
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper" (no "onboard-" prefix) is OpenAI's CLOUD realtime model
|
||||
// (gpt-realtime-whisper); "onboard-whisper" is the on-device sherpa-onnx
|
||||
// Whisper backend. Same underlying model family, two different places it
|
||||
// runs -- see the same note in native/voice-control.ts. The "whisper" value
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device",
|
||||
onboard: "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "Whisper",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
};
|
||||
|
||||
@@ -537,7 +544,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "elevenlabs", "whisper", "soniox"],
|
||||
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -545,7 +552,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).",
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
|
||||
@@ -13,12 +13,13 @@ import {
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
import {
|
||||
ASR_MODEL,
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
cancelAsrModelDownload,
|
||||
deleteAsrModel,
|
||||
onAsrModelStateChanged,
|
||||
startAsrModelDownload,
|
||||
type AsrModelId,
|
||||
} from "../../native/asr-model";
|
||||
import { TextViewerLayer } from "../../apps/files/text-viewer";
|
||||
import type { LayerContext } from "../layers";
|
||||
@@ -119,7 +120,8 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
asrModelMenuItem(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -344,48 +346,52 @@ function localModelMenuItem(): MenuItem {
|
||||
};
|
||||
}
|
||||
|
||||
const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`;
|
||||
function asrModelMb(id: AsrModelId): string {
|
||||
return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`;
|
||||
}
|
||||
|
||||
let asrModelRenderUnsub: (() => void) | null = null;
|
||||
const asrModelRenderUnsub: Partial<Record<AsrModelId, () => void>> = {};
|
||||
|
||||
function watchAsrModelDownload(ctx: LayerContext): void {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = onAsrModelStateChanged((state) => {
|
||||
function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void {
|
||||
asrModelRenderUnsub[id]?.();
|
||||
asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => {
|
||||
ctx.actions.requestRender();
|
||||
if (state.status !== "downloading") {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = null;
|
||||
asrModelRenderUnsub[id]?.();
|
||||
asrModelRenderUnsub[id] = undefined;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function asrModelStatusText(): string {
|
||||
const state = asrModelState();
|
||||
function asrModelStatusText(id: AsrModelId): string {
|
||||
const state = asrModelState(id);
|
||||
if (state.status === "ready") return "downloaded";
|
||||
if (state.status === "downloading") {
|
||||
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
|
||||
return `${pct}% of ${ASR_MODEL_MB}`;
|
||||
return `${pct}% of ${asrModelMb(id)}`;
|
||||
}
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-device transcription model. */
|
||||
function asrModelMenuItem(): MenuItem {
|
||||
/** Download/cancel/delete management for one on-device transcription model. */
|
||||
function asrModelMenuItem(id: AsrModelId): MenuItem {
|
||||
const def = ASR_MODELS[id];
|
||||
const rowLabel = `On-device model: ${def.label}`;
|
||||
return {
|
||||
label: "On-device voice model",
|
||||
label: rowLabel,
|
||||
description:
|
||||
`${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` +
|
||||
`${def.label} (${asrModelMb(id)} download). ` +
|
||||
"Transcribes voice input on the phone itself, with no API key or cloud service. " +
|
||||
"Required for the On-device transcription provider; the cloud providers work without it. " +
|
||||
"Required for its matching Transcription Provider option; the other providers work without it. " +
|
||||
"An interrupted download resumes where it left off.",
|
||||
onSelect: (ctx) => {
|
||||
const state = asrModelState();
|
||||
const state = asrModelState(id);
|
||||
const action: MenuItem =
|
||||
state.status === "downloading"
|
||||
? {
|
||||
label: "Cancel download",
|
||||
onSelect: (innerCtx) => {
|
||||
cancelAsrModelDownload();
|
||||
cancelAsrModelDownload(id);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
@@ -393,22 +399,22 @@ function asrModelMenuItem(): MenuItem {
|
||||
? {
|
||||
label: "Delete model",
|
||||
onSelect: (innerCtx) => {
|
||||
deleteAsrModel();
|
||||
deleteAsrModel(id);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: {
|
||||
label: `Download (${ASR_MODEL_MB})`,
|
||||
label: `Download (${asrModelMb(id)})`,
|
||||
onSelect: (innerCtx) => {
|
||||
startAsrModelDownload();
|
||||
watchAsrModelDownload(innerCtx);
|
||||
startAsrModelDownload(id);
|
||||
watchAsrModelDownload(id, innerCtx);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
};
|
||||
openModalMenu(ctx, "On-device voice model", [action], 0);
|
||||
openModalMenu(ctx, rowLabel, [action], 0);
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText());
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ export type LayerActions = {
|
||||
* (hands-free); otherwise it runs until stopVoiceCapture.
|
||||
*/
|
||||
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
startContinuousVoiceCapture: () => Promise<void> | void;
|
||||
|
||||
+56
-16
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
|
||||
import { Layer, type LayerActions, type LayerContext } from "../layers";
|
||||
import { paintInputDialog } from "./input-dialog";
|
||||
// After the mic stops, the provider's final transcript can trail in (cloud
|
||||
// commit round-trip); wait this long for it before refining with what we have.
|
||||
// After native recognition finishes, a cloud final can still trail in.
|
||||
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
|
||||
|
||||
/**
|
||||
@@ -80,6 +79,10 @@ export class VoiceInputLayer implements Layer {
|
||||
private capturing = false;
|
||||
/** Continuation stopped; waiting for the trailing final transcript. */
|
||||
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private pendingFollowup = false;
|
||||
private stoppingCapture = false;
|
||||
private stopGeneration = 0;
|
||||
private acceptingTranscript = true;
|
||||
private refineHandle: AnthropicStreamHandle | null = null;
|
||||
private menuIndex = 0;
|
||||
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
|
||||
@@ -155,21 +158,48 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
this.capturing = false;
|
||||
this.phase = "menu";
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.autoSend && this.sendTargets.length) {
|
||||
// Skip-confirmation (wakeword): send to the default target as soon as the
|
||||
// transcript finalizes, or after a short wait for the trailing final.
|
||||
this.pendingAutoSend = true;
|
||||
this.status = "Sending...";
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
} else if (this.status.endsWith("...")) {
|
||||
// A progress status ("Starting microphone...", "Listening...") gives
|
||||
// way to the menu prompt; an error (ending in ".") stays visible.
|
||||
this.status = "Send, continue, or discard?";
|
||||
}
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingAutoSend) {
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Native stop can outlast its synchronous join while Whisper is decoding. */
|
||||
private stopCapture(onStopped: () => void): void {
|
||||
const generation = ++this.stopGeneration;
|
||||
this.stoppingCapture = true;
|
||||
const afterStop = () => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
onStopped();
|
||||
this.actions.requestRender();
|
||||
};
|
||||
const stopped = this.actions.stopVoiceCapture();
|
||||
if (!stopped) {
|
||||
afterStop();
|
||||
return;
|
||||
}
|
||||
void stopped.then(afterStop, (error) => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
this.pendingAutoSend = false;
|
||||
this.acceptingTranscript = false;
|
||||
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
|
||||
});
|
||||
}
|
||||
|
||||
/** Fire the queued skip-confirmation send (or fall back to the menu). */
|
||||
private performAutoSend(): void {
|
||||
if (this.autoSendTimer !== null) {
|
||||
@@ -193,7 +223,7 @@ export class VoiceInputLayer implements Layer {
|
||||
/** The menu rows: one per send target, then Continue, then Discard. */
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasText = text.length > 0 && !this.stoppingCapture;
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
@@ -201,6 +231,7 @@ export class VoiceInputLayer implements Layer {
|
||||
label: target.label,
|
||||
dim: !hasText,
|
||||
onSelect: () => {
|
||||
if (this.stoppingCapture) return;
|
||||
this.dismiss();
|
||||
if (hasText) target.onSend(text);
|
||||
},
|
||||
@@ -208,9 +239,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
rows.push({
|
||||
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
dim: !hasLlmKey || this.stoppingCapture,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
|
||||
},
|
||||
});
|
||||
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
|
||||
@@ -222,7 +253,7 @@ export class VoiceInputLayer implements Layer {
|
||||
const inMenu = this.phase === "menu";
|
||||
paintInputDialog(image, {
|
||||
title: this.capturing && this.listening ? "Voice ●" : "Voice",
|
||||
status: this.status,
|
||||
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
|
||||
text: this.displayText() || this.placeholderText(),
|
||||
rows: inMenu ? this.menuRows() : [],
|
||||
selectedRow: this.menuIndex,
|
||||
@@ -284,6 +315,7 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Continue selected: keep the message aside and record a follow-up. */
|
||||
private startContinuation(): void {
|
||||
this.acceptingTranscript = true;
|
||||
this.baseText = this.displayText().trim();
|
||||
this.finalizedText = "";
|
||||
this.liveText = "";
|
||||
@@ -298,15 +330,18 @@ export class VoiceInputLayer implements Layer {
|
||||
private endContinuationCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.status = "Refining...";
|
||||
this.pendingFollowup = true;
|
||||
this.status = "Finishing transcription...";
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingFollowup) {
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
// The provider's committed transcript arrives shortly after stop; refine
|
||||
// when it does, or after a timeout with whatever partials we have.
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private beginRefine(): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -343,9 +378,10 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
|
||||
private cancelContinuation(status: string): void {
|
||||
this.acceptingTranscript = false;
|
||||
if (this.capturing) {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.stopCapture(() => {});
|
||||
}
|
||||
this.refineHandle?.cancel();
|
||||
this.refineHandle = null;
|
||||
@@ -353,6 +389,7 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
private backToMenu(text: string, status: string): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -366,6 +403,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
++this.stopGeneration;
|
||||
this.acceptingTranscript = false;
|
||||
this.pendingFollowup = false;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
@@ -428,14 +468,14 @@ export class VoiceInputLayer implements Layer {
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
// The refine stream owns the text buffers once it starts; a transcript
|
||||
// that trails in after that point is stale.
|
||||
if (this.phase === "refining") return;
|
||||
if (!this.acceptingTranscript || this.phase === "refining") return;
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
|
||||
if (this.phase === "continuing" && this.pendingFollowup) {
|
||||
// The follow-up finalized; no need to keep waiting.
|
||||
this.beginRefine();
|
||||
return;
|
||||
|
||||
+30
-2
@@ -125,7 +125,7 @@ test('synchronous provider errors do not leave a permanently active turn', () =>
|
||||
assert.equal(session.status, 'failed');
|
||||
});
|
||||
|
||||
function draftEnv({ prepare = async () => true, start = async () => {}, initialStatus = 'Ready' } = {}) {
|
||||
function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) {
|
||||
const transcripts = new Set(), statuses = new Set(), timers = new Map();
|
||||
const sent = [];
|
||||
let starts = 0, stops = 0, mic = false, timerId = 0;
|
||||
@@ -139,7 +139,7 @@ function draftEnv({ prepare = async () => true, start = async () => {}, initialS
|
||||
setTimeout(cb) { timers.set(++timerId, cb); return timerId; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
});
|
||||
const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; } }, prepare, () => {}, (text) => sent.push(text));
|
||||
const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text));
|
||||
return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic,
|
||||
transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); },
|
||||
status(status) { for (const cb of [...statuses]) cb({ status }); },
|
||||
@@ -174,6 +174,34 @@ test('release before permission resolves never starts a microphone', async () =>
|
||||
assert.deepEqual(env.sent, []);
|
||||
});
|
||||
|
||||
test('chat waits past its fallback deadline for native recognition', async () => {
|
||||
let complete;
|
||||
const stopped = new Promise(resolve => { complete = resolve; });
|
||||
const env = draftEnv({ stop: () => stopped });
|
||||
await env.draft.start();
|
||||
env.transcript('first segment');
|
||||
env.draft.release(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.draft.phase, 'finishing');
|
||||
env.transcript('first segment and the rest', true);
|
||||
complete(); await stopped; env.timeout();
|
||||
assert.deepEqual(env.sent, ['first segment and the rest']);
|
||||
});
|
||||
|
||||
test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => {
|
||||
let complete;
|
||||
const stopped = new Promise(resolve => { complete = resolve; });
|
||||
const env = draftEnv({ stop: () => stopped });
|
||||
await env.draft.start();
|
||||
env.draft.release(); env.draft.cancel();
|
||||
await env.draft.start();
|
||||
env.transcript('new speech');
|
||||
complete(); await stopped; env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.draft.phase, 'listening');
|
||||
env.draft.cancel();
|
||||
});
|
||||
|
||||
test('release during async mic startup stops capture after it starts', async () => {
|
||||
let started;
|
||||
const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) });
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import java.lang.reflect.*;
|
||||
import java.util.*;
|
||||
|
||||
public class TextureCacheProtocolTest {
|
||||
static int u16(byte[] b, int p) { return (b[p]&255) | (b[p+1]&255)<<8; }
|
||||
static int u32(byte[] b, int p) { return u16(b,p) | u16(b,p+2)<<16; }
|
||||
public static void main(String[] args) throws Exception {
|
||||
TextureCacheState cache = new TextureCacheState();
|
||||
byte[] pixels = new byte[255*255];
|
||||
for (int i=0;i<pixels.length;i++) pixels[i]=(byte)(i%15+1);
|
||||
ImageAtlas.Entry big = new ImageAtlas.Entry(255,255,pixels);
|
||||
for (int i=0;i<3;i++) assert cache.ensureImage(i,big)>=0;
|
||||
ImageAtlas.Entry tiny = new ImageAtlas.Entry(1,1,new byte[]{15});
|
||||
int imageOffset = cache.ensureImage(100,tiny);
|
||||
assert imageOffset>65535;
|
||||
GlyphAtlas.Glyph glyph = new GlyphAtlas.Glyph(1,1,0,1,0,null,new byte[]{15});
|
||||
int glyphOffset = cache.ensureGlyph(42,'A',glyph);
|
||||
int table = cache.fontTableOffset(42);
|
||||
assert table>65535 && glyphOffset==table+96*4;
|
||||
byte[] memory = new byte[262144];
|
||||
for (byte[] upload: cache.drainUploadPayloads(3600)) {
|
||||
assert upload[0]==18;
|
||||
int pos=1;
|
||||
while (pos<upload.length) {
|
||||
int offset=u32(upload,pos), n=u16(upload,pos+4);
|
||||
System.arraycopy(upload,pos+6,memory,offset,n); pos+=6+n;
|
||||
}
|
||||
assert pos==upload.length;
|
||||
}
|
||||
assert memory[imageOffset]==1 && memory[imageOffset+2]==0x1f;
|
||||
assert u32(memory,table+('A'-32)*4)==glyphOffset;
|
||||
assert !cache.hasPendingUploads();
|
||||
// Exercise the production planner encoders with offsets above 64 KiB.
|
||||
Class<?> selected=Class.forName("com.faceclaw.app.TexturePlanner$Selected");
|
||||
Constructor<?> ctor=selected.getDeclaredConstructors()[0]; ctor.setAccessible(true);
|
||||
Object image=ctor.newInstance(SurfaceCompositor.ScreenDraw.image(100,321,123),null,tiny,321,15);
|
||||
Method encode=TexturePlanner.class.getDeclaredMethod("encodeImageDraw",TextureCacheState.class,selected);
|
||||
encode.setAccessible(true);
|
||||
byte[] draw=(byte[])encode.invoke(null,cache,image);
|
||||
assert draw.length==10 && draw[0]==19 && u32(draw,1)==imageOffset;
|
||||
assert u16(draw,5)==321 && u16(draw,7)==123 && draw[9]==31;
|
||||
Object letter=ctor.newInstance(SurfaceCompositor.ScreenDraw.glyph(42,'A',321,123,255),glyph,null,321,15);
|
||||
Method runs=TexturePlanner.class.getDeclaredMethod("buildRuns",TextureCacheState.class,List.class,List.class);
|
||||
runs.setAccessible(true); List<byte[]> out=new ArrayList<>();
|
||||
runs.invoke(null,cache,Collections.singletonList(letter),out);
|
||||
byte[] run=out.get(0);
|
||||
assert run.length==12 && run[0]==20 && u32(run,1)==table;
|
||||
assert u16(run,5)==321 && u16(run,7)==123 && run[9]==31 && run[10]==1 && run[11]=='A';
|
||||
int id=200;
|
||||
while (cache.ensureImage(id++,big)>=0) {}
|
||||
int before=cache.usedBytes();
|
||||
assert cache.ensureImage(id,big)==-1 && cache.usedBytes()==before;
|
||||
cache.reset(); assert cache.usedBytes()==0 && !cache.hasPendingUploads();
|
||||
assert cache.ensureImage(id,tiny)==0;
|
||||
}
|
||||
}
|
||||
@@ -47,15 +47,16 @@ test("dismiss hides from any state", () => {
|
||||
assert.deepEqual(reduceGlance(shown, { type: "dismiss" }, T0), GLANCE_HIDDEN);
|
||||
});
|
||||
|
||||
test("gesture mapping: taps and head-tilt press, long-press holds, double-tap dismisses only a visible board", () => {
|
||||
test("gesture mapping: taps and head-tilt press, both long-press gestures hold, double-tap dismisses only a visible board", () => {
|
||||
assert.deepEqual(glanceEventForGesture("click", false), { type: "press" });
|
||||
assert.deepEqual(glanceEventForGesture("head-tilt", false), { type: "press" });
|
||||
assert.deepEqual(glanceEventForGesture("long-press", false), { type: "hold" });
|
||||
assert.deepEqual(glanceEventForGesture("short-then-long-press", false), { type: "hold" });
|
||||
assert.deepEqual(glanceEventForGesture("short-then-long-press", true), { type: "hold" });
|
||||
assert.deepEqual(glanceEventForGesture("long-press-release", true), { type: "release" });
|
||||
assert.equal(glanceEventForGesture("double-click", false), null);
|
||||
assert.deepEqual(glanceEventForGesture("double-click", true), { type: "dismiss" });
|
||||
assert.equal(glanceEventForGesture("scroll-up", true), null);
|
||||
assert.equal(glanceEventForGesture("short-then-long-press", true), null);
|
||||
});
|
||||
|
||||
test("the press timeout is the host's when supplied", () => {
|
||||
|
||||
@@ -57,3 +57,25 @@ test("dividers: left edges of the right column, top edges of the bottom row, min
|
||||
{ x0: 288, y0: 144, x1: 575, y1: 144 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("2x3 adds a bottom row without moving or resizing the first four slots", () => {
|
||||
const { SIX_SLOT_LAYOUT } = require("../.test-build/app/apps/glanceboard/layout.js");
|
||||
assert.equal(SIX_SLOT_LAYOUT.width, 576);
|
||||
assert.equal(SIX_SLOT_LAYOUT.height, 432);
|
||||
assert.deepEqual(SIX_SLOT_LAYOUT.slots.slice(0, 4).map((slot) => slot.rect), QUADRANT_LAYOUT.slots.map((slot) => slot.rect));
|
||||
assert.deepEqual(SIX_SLOT_LAYOUT.slots.slice(4).map((slot) => slot.rect), [
|
||||
{ x: 0, y: 288, width: 288, height: 144 },
|
||||
{ x: 288, y: 288, width: 288, height: 144 },
|
||||
]);
|
||||
const regions = resolveGlanceRegions(SIX_SLOT_LAYOUT, ["none", "none", "terminal", "none", "terminal", "music"], spans);
|
||||
assert.deepEqual(regions, [
|
||||
{ choice: "terminal", rect: { x: 0, y: 144, width: 288, height: 288 }, slots: [2, 4] },
|
||||
{ choice: "music", rect: { x: 288, y: 288, width: 288, height: 144 }, slots: [5] },
|
||||
]);
|
||||
const { slotDividers } = require("../.test-build/app/apps/glanceboard/layout.js");
|
||||
const bottomDividers = slotDividers(SIX_SLOT_LAYOUT, regions).filter((line) => line.y0 === 288);
|
||||
assert.deepEqual(bottomDividers, [
|
||||
{ x0: 288, y0: 288, x1: 288, y1: 431 },
|
||||
{ x0: 288, y0: 288, x1: 575, y1: 288 },
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
const layout = require('../.test-build/app/apps/glanceboard/layout.js');
|
||||
|
||||
function load(file, dependencies) {
|
||||
const context = { exports: {}, console, require: (name) => {
|
||||
assert.ok(name in dependencies, name);
|
||||
return dependencies[name];
|
||||
} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, context);
|
||||
return context.exports;
|
||||
}
|
||||
|
||||
function harness() {
|
||||
const values = new Map();
|
||||
const listeners = new Set();
|
||||
class Setting {
|
||||
constructor(options) { Object.assign(this, options); }
|
||||
get() { return values.get(this.storageKey) ?? this.defaultValue; }
|
||||
set(value) { values.set(this.storageKey, value); }
|
||||
}
|
||||
const config = {
|
||||
ConfigSettingBoolean: Setting, ConfigSettingEnum: Setting,
|
||||
onAnySettingChanged: (listener) => { listeners.add(listener); return () => listeners.delete(listener); },
|
||||
};
|
||||
const widgets = {
|
||||
glanceWidgetSpans: (choice) => choice === 'calendar' || choice === 'terminal',
|
||||
findGlanceWidget: () => ({ create: () => ({ start() {}, stop() {}, paint() {} }) }),
|
||||
};
|
||||
const settings = load('app/apps/glanceboard/glanceboard-settings.ts', {
|
||||
'../../ui/dashboard-settings': config, './layout': layout, './widgets': widgets,
|
||||
});
|
||||
class GrayImage {
|
||||
constructor(width, height) { this.width = width; this.height = height; }
|
||||
drawLine() {}
|
||||
composeInto() {}
|
||||
}
|
||||
const { GlanceBoard } = load('app/apps/glanceboard/board.ts', {
|
||||
'../../graphics/image': { GrayImage }, '../../ui/dashboard-settings': config,
|
||||
'./glanceboard-settings': settings, './layout': layout, './widgets': widgets,
|
||||
});
|
||||
return { settings, GlanceBoard, values, listeners, notify: () => [...listeners].forEach((listener) => listener()) };
|
||||
}
|
||||
|
||||
test('layout defaults to 2x2; expanding preserves stored slots and remembers the extra row', () => {
|
||||
const { settings: s, values } = harness();
|
||||
values.set('glanceboard.quadrants.slot.0', 'compass');
|
||||
assert.equal(s.glanceLayoutSetting.get(), '2x2');
|
||||
assert.equal(s.glanceLayout(), layout.QUADRANT_LAYOUT);
|
||||
s.glanceLayoutSetting.set('2x3');
|
||||
assert.equal(s.glanceLayout(), layout.SIX_SLOT_LAYOUT);
|
||||
assert.deepEqual(Array.from(s.glanceSlotSettings(), (slot) => slot.get()),
|
||||
['compass', 'calendar', 'music', 'calendar', 'none', 'none']);
|
||||
s.glanceSlotSettings()[5].set('nightscout');
|
||||
s.glanceLayoutSetting.set('2x2');
|
||||
assert.equal(s.glanceSlotSettings().length, 4);
|
||||
s.glanceLayoutSetting.set('2x3');
|
||||
assert.equal(s.glanceSlotSettings()[5].get(), 'nightscout');
|
||||
});
|
||||
|
||||
test('a spanning widget in the middle row keeps only one adjacent partner', () => {
|
||||
const { settings: s } = harness();
|
||||
s.glanceLayoutSetting.set('2x3');
|
||||
const slots = s.glanceSlotSettings();
|
||||
for (const index of [0, 2, 4]) slots[index].set('terminal');
|
||||
s.clearConflictingSlots(s.glanceLayout(), 2, 'terminal');
|
||||
assert.deepEqual(Array.from(slots, (slot) => slot.get()),
|
||||
['terminal', 'calendar', 'terminal', 'calendar', 'none', 'none']);
|
||||
slots[4].set('terminal');
|
||||
s.clearConflictingSlots(s.glanceLayout(), 4, 'terminal');
|
||||
assert.equal(slots[0].get(), 'none');
|
||||
assert.equal(slots[2].get(), 'terminal');
|
||||
});
|
||||
|
||||
test('an active empty board resizes and requests a frame when the layout changes', () => {
|
||||
const h = harness();
|
||||
for (const slot of h.settings.glanceSlotSettings(layout.SIX_SLOT_LAYOUT)) slot.set('none');
|
||||
let renders = 0;
|
||||
const board = new h.GlanceBoard(() => renders++);
|
||||
board.start();
|
||||
assert.equal(board.paint().height, 288);
|
||||
h.settings.glanceLayoutSetting.set('2x3');
|
||||
h.notify();
|
||||
assert.equal(board.paint().width, 576);
|
||||
assert.equal(board.paint().height, 432);
|
||||
assert.equal(renders, 1);
|
||||
h.settings.glanceLayoutSetting.set('2x2');
|
||||
h.notify();
|
||||
assert.equal(board.paint().height, 288);
|
||||
assert.equal(renders, 2);
|
||||
board.stop();
|
||||
assert.equal(h.listeners.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
const test = require('node:test');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
|
||||
test('256 KiB cache uploads and planner draws preserve wide image and glyph offsets', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-texture-java-'));
|
||||
const javaBin = (name) => process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', name) : name;
|
||||
const root = path.join(__dirname, '../App_Resources/Android/src/main/java/com/faceclaw/app');
|
||||
try {
|
||||
const clock = path.join(directory, 'SystemClock.java');
|
||||
const log = path.join(directory, 'Log.java');
|
||||
fs.writeFileSync(clock, 'package android.os; public class SystemClock { public static long elapsedRealtime() { return 0; } }');
|
||||
fs.writeFileSync(log, 'package android.util; public class Log { public static int i(String tag, String msg) { return 0; } }');
|
||||
const sources = ['GlyphAtlas.java', 'ImageAtlas.java', 'FwGlyphAtlas.java', 'SurfaceCompositor.java',
|
||||
'util/BmpUtil.java', 'util/CollectionUtils.java', 'g2protocol/BleProtocol.java',
|
||||
'g2protocol/BleImageOptimizer.java', 'g2protocol/TextureCacheState.java', 'g2protocol/TexturePlanner.java'];
|
||||
execFileSync(javaBin('javac'), ['-d', directory, ...sources.map(file => path.join(root, file)),
|
||||
clock, log, path.join(__dirname, 'fixtures/TextureCacheProtocolTest.java')], { stdio: 'pipe' });
|
||||
execFileSync(javaBin('java'), ['-ea', '-cp', directory, 'com.faceclaw.app.TextureCacheProtocolTest'], { stdio: 'pipe' });
|
||||
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, deps, globals = {}) {
|
||||
const exports = {};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, { exports, console, ...globals, require: name => deps[name] ?? {} });
|
||||
return exports;
|
||||
}
|
||||
|
||||
function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
|
||||
const listeners = new Set(), timers = new Map(), sent = [], refinements = [];
|
||||
let complete, timerId = 0, starts = 0, frame;
|
||||
let completion = new Promise(resolve => { complete = resolve; });
|
||||
const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', {
|
||||
'../../native/voice-control': { voiceControlBridge: {
|
||||
onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); },
|
||||
onStatus() { return () => {}; },
|
||||
} },
|
||||
'../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } },
|
||||
'../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } },
|
||||
'../gestures': { gestureHints: () => '' },
|
||||
'./input-dialog': { paintInputDialog(_image, options) { frame = options; } },
|
||||
}, {
|
||||
setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
});
|
||||
const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); };
|
||||
const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(),
|
||||
actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() {
|
||||
if (synchronousFinal) transcript(synchronousFinal, true);
|
||||
return deferred ? completion : undefined;
|
||||
} },
|
||||
sendTargets: [{ id: 'send', label: 'Send', onSend: text => sent.push(text) }],
|
||||
});
|
||||
layer.startCapture();
|
||||
return { layer, sent, refinements, transcript, starts: () => starts,
|
||||
input: type => layer.handleInput({ type }, {}),
|
||||
frame() { layer.paint({}, () => ({})); return frame; },
|
||||
timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } },
|
||||
async complete() { complete(); await completion; await Promise.resolve(); },
|
||||
nextCapture() { completion = new Promise(resolve => { complete = resolve; }); },
|
||||
timers: () => timers.size,
|
||||
};
|
||||
}
|
||||
|
||||
for (const partial of ['', 'Set a timer']) {
|
||||
test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
if (partial) env.transcript(partial);
|
||||
env.layer.endCapture();
|
||||
env.timeout(); // well beyond the old fallback deadline
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.timers(), 0);
|
||||
env.transcript('Set a timer for five minutes', true);
|
||||
await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, ['Set a timer for five minutes']);
|
||||
assert.equal(env.timers(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
async function continuation() {
|
||||
const env = dialog();
|
||||
env.layer.endCapture();
|
||||
env.transcript('Set a timer for five minutes', true);
|
||||
await env.complete();
|
||||
env.nextCapture();
|
||||
env.input('scroll-down'); env.input('click'); // Continue
|
||||
env.input('click'); // stop follow-up
|
||||
return env;
|
||||
}
|
||||
|
||||
test('Continue refines the complete late result instead of appending an editing instruction', async () => {
|
||||
const env = await continuation();
|
||||
env.timeout();
|
||||
assert.equal(env.refinements.length, 0);
|
||||
env.transcript('Change five to ten', true);
|
||||
await env.complete(); env.timeout();
|
||||
assert.equal(env.refinements.length, 1);
|
||||
assert.equal(env.refinements[0].original, 'Set a timer for five minutes');
|
||||
assert.equal(env.refinements[0].followup, 'Change five to ten');
|
||||
});
|
||||
|
||||
test('Send and Continue cannot consume partial text while native recognition is pending', async () => {
|
||||
const env = dialog();
|
||||
env.transcript('Set a timer'); env.layer.endCapture();
|
||||
assert.equal(env.frame().status, 'Finishing transcription...');
|
||||
env.input('click'); env.input('scroll-down'); env.input('click');
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.starts(), 1);
|
||||
env.transcript('Set a timer for five minutes', true); await env.complete();
|
||||
env.input('scroll-up'); env.input('click');
|
||||
assert.deepEqual(env.sent, ['Set a timer for five minutes']);
|
||||
});
|
||||
|
||||
test('cancelling a pending continuation ignores its late transcript', async () => {
|
||||
const env = await continuation();
|
||||
env.input('double-click');
|
||||
env.transcript('Change five to ten', true); await env.complete(); env.timeout();
|
||||
assert.equal(env.refinements.length, 0);
|
||||
assert.equal(env.frame().text, 'Set a timer for five minutes');
|
||||
});
|
||||
|
||||
test('closing while native recognition is pending never schedules a fallback or sends', async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
env.layer.endCapture(); env.layer.onRemoved();
|
||||
env.transcript('late', true); await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.timers(), 0);
|
||||
});
|
||||
|
||||
test('failed or empty native captures return to the menu without sending', async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
env.layer.endCapture(); await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.frame().status, 'Send, continue, or discard?');
|
||||
});
|
||||
|
||||
test('cloud fallback and a synchronous final still send exactly once', () => {
|
||||
for (const synchronousFinal of [undefined, 'complete']) {
|
||||
const env = dialog({ autoSend: true, deferred: false, synchronousFinal });
|
||||
env.transcript('partial'); env.layer.endCapture(); env.timeout();
|
||||
assert.deepEqual(env.sent, [synchronousFinal ?? 'partial']);
|
||||
}
|
||||
});
|
||||
|
||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||
let controller;
|
||||
class Controller {
|
||||
constructor() { controller = this; }
|
||||
setListener(listener) { this.listener = listener; }
|
||||
setCommunicator() {} setUsePhoneMic() {} setSaveRecordings() {} setEndpointing() {}
|
||||
setNoiseSuppression() {} setBeamFilter() {} clearSpeakerVerification() {} setOnboardModelKind() {}
|
||||
start(_mode, id) { this.captureId = id; this.active = true; }
|
||||
stop() { this.active = false; }
|
||||
isCapturing() { return this.active; }
|
||||
}
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: Controller,
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
} } } });
|
||||
const bridge = new FaceclawVoiceControlBridge();
|
||||
const options = { provider: 'onboard-whisper' };
|
||||
const events = [];
|
||||
bridge.onTranscript(event => events.push(event.text));
|
||||
bridge.startPushToTalk(options);
|
||||
const stopped = bridge.stopPushToTalk().then(() => events.push('stopped'));
|
||||
controller.listener.onStopped(0); // an older raw/cloud capture's callback
|
||||
await Promise.resolve(); assert.deepEqual(events, []);
|
||||
controller.listener.onTranscript('complete', true);
|
||||
assert.deepEqual(events, ['complete']);
|
||||
controller.listener.onStopped(controller.captureId); await stopped;
|
||||
assert.deepEqual(events, ['complete', 'stopped']);
|
||||
bridge.startContinuousCapture(options); bridge.startPushToTalk(options);
|
||||
assert.equal(bridge.stopPushToTalk(), undefined);
|
||||
assert.equal(controller.active, true);
|
||||
bridge.stopContinuousCapture(); controller.listener.onStopped(controller.captureId);
|
||||
});
|
||||
Reference in new issue
Block a user