Merge branch 'main' into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-09-18 00:08:02 -07:00
commit edef7138fd
29 files changed
+1089 -234

No files matched your search

+14 -3
View File
@@ -89,11 +89,22 @@ export class VoiceDraft {
}
private stopAndFinalize(): void {
// Install fallback first: stop may synchronously deliver the final.
this.timer = setTimeout(() => this.finish(), 1500);
const generation = this.generation;
const afterStop = () => {
if (generation !== this.generation || !this.isFinishing()) return;
this.timer = setTimeout(() => this.finish(), 1500);
};
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
// Native recognition can take longer than the synchronous stop join.
// Its final callback may finish the draft before this promise resolves.
const stopped = this.actions.stopVoiceCapture();
if (!stopped) { afterStop(); return; }
void stopped.then(afterStop, (error) => {
if (generation === this.generation) this.cancel(String(error));
});
} else {
afterStop();
}
}
+15 -5
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { onAnySettingChanged } from "../../ui/dashboard-settings";
import { glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings";
import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout";
import { glanceLayout, glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings";
import { resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout";
import { type GlanceWidget } from "./widget";
import { findGlanceWidget, glanceWidgetSpans } from "./widgets";
@@ -27,11 +27,18 @@ export class GlanceBoard {
private regions: LiveRegion[] = [];
private started = false;
private unsubscribeSettings: (() => void) | null = null;
private currentLayout: GlanceLayout;
constructor(
private readonly requestRender: () => void,
readonly layout: GlanceLayout = QUADRANT_LAYOUT,
) {}
private readonly fixedLayout?: GlanceLayout,
) {
this.currentLayout = fixedLayout ?? glanceLayout();
}
get layout(): GlanceLayout {
return this.currentLayout;
}
get width(): number {
return this.layout.width;
@@ -65,9 +72,12 @@ export class GlanceBoard {
/** Rebuild the live regions from the settings, keeping widgets whose region is unchanged. */
private applySlotSettings(): void {
if (!this.started) return;
const layout = this.fixedLayout ?? glanceLayout();
const layoutChanged = layout !== this.currentLayout;
this.currentLayout = layout;
const previous = new Map(this.regions.map((live) => [live.key, live]));
const next: LiveRegion[] = [];
let changed = false;
let changed = layoutChanged;
for (const region of this.currentRegions()) {
const key = regionKey(region);
const kept = previous.get(key);
+51 -21
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { type InputEvent } from "../../ui/gestures";
import { type Layer, type LayerContext, type PaintBelow } from "../../ui/layers";
import { drawSelectionHighlight, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu";
@@ -12,6 +12,8 @@ import { GlanceBoard } from "./board";
import {
clearConflictingSlots,
glanceboardEnabledSetting,
glanceLayout,
glanceLayoutSetting,
glanceShowLinesSetting,
glanceShowOnHeadTiltSetting,
glanceShowOnLongPressSetting,
@@ -20,7 +22,7 @@ import {
glanceTapDurationSetting,
type GlanceSlotChoice,
} from "./glanceboard-settings";
import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers } from "./layout";
import { resolveGlanceRegions, slotDividers } from "./layout";
import { glanceWidgetSpans } from "./widgets";
export const GLANCEBOARD_WINDOW_ID = "glanceboard";
@@ -49,24 +51,33 @@ function pageMenuLayout(width: number) {
* and the menu (enable toggle, live preview, slot contents, settings). Right
* half: a scaled preview of the board's slots with their dividers and the
* name of the widget in each. "Select Contents" moves focus into that
* preview, where scroll picks a quadrant and a tap opens the picker for it;
* preview, where scroll picks a slot and a tap opens the picker for it;
* a double-tap returns focus to the menu.
*/
class GlanceboardHomeLayer implements Layer {
private menu: MenuLayer | null = null;
private menuTop = 0;
private menuHeight = 0;
private focus: "menu" | "preview" = "menu";
private selectedSlot = 0;
constructor(private readonly setPreviewVisible: (visible: boolean) => void) {}
private currentMenu(ctx: LayerContext, menuTop: number): MenuLayer {
if (!this.menu || this.menuTop !== menuTop) {
const { width, height } = ctx.stack.getBaseSize();
const { width, height } = ctx.stack.getBaseSize();
if (!this.menu || this.menuTop !== menuTop || this.menuHeight !== height) {
const half = (width / 2) | 0;
const items: MenuItem[] = [
toggleSettingMenuItem(glanceboardEnabledSetting),
{
label: "Preview",
onSelect: (menuCtx) => menuCtx.stack.push(new GlancePreviewLayer(() => menuCtx.actions.requestRender())),
onSelect: (menuCtx) => {
menuCtx.stack.push(new GlancePreviewLayer(
() => menuCtx.actions.requestRender(),
() => this.setPreviewVisible(false),
));
this.setPreviewVisible(true);
},
},
{
label: "Select Contents",
@@ -80,6 +91,7 @@ class GlanceboardHomeLayer implements Layer {
},
];
this.menuTop = menuTop;
this.menuHeight = height;
this.menu = new MenuLayer(null, items, {
x: 8, y: menuTop, width: half - 16,
showBorder: false, minHeight: 0, maxHeight: height - menuTop,
@@ -92,9 +104,10 @@ class GlanceboardHomeLayer implements Layer {
private previewRect(width: number, height: number): { x: number; y: number; width: number; height: number } {
const half = (width / 2) | 0;
const availableWidth = width - half - 2 * PREVIEW_MARGIN;
const scale = Math.min(availableWidth / QUADRANT_LAYOUT.width, (height - 2 * PREVIEW_MARGIN) / QUADRANT_LAYOUT.height);
const previewWidth = Math.round(QUADRANT_LAYOUT.width * scale);
const previewHeight = Math.round(QUADRANT_LAYOUT.height * scale);
const layout = glanceLayout();
const scale = Math.min(availableWidth / layout.width, (height - 2 * PREVIEW_MARGIN) / layout.height);
const previewWidth = Math.round(layout.width * scale);
const previewHeight = Math.round(layout.height * scale);
return {
x: half + PREVIEW_MARGIN + (((availableWidth - previewWidth) / 2) | 0),
y: ((height - previewHeight) / 2) | 0,
@@ -123,6 +136,7 @@ class GlanceboardHomeLayer implements Layer {
const menuCtx: LayerContext = { ...ctx, stack: Object.assign(Object.create(ctx.stack), { isFocused: () => menuFocused }) };
menu.paint(menuCtx, () => image);
this.selectedSlot = Math.min(this.selectedSlot, glanceLayout().slots.length - 1);
const preview = this.previewRect(width, height);
drawSlotPreview(image, preview, {
selectedSlot: this.selectedSlot,
@@ -133,7 +147,8 @@ class GlanceboardHomeLayer implements Layer {
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
if (this.focus === "preview") {
const slotCount = QUADRANT_LAYOUT.slots.length;
const slotCount = glanceLayout().slots.length;
this.selectedSlot = Math.min(this.selectedSlot, slotCount - 1);
switch (event.type) {
case "scroll-up":
this.selectedSlot = (this.selectedSlot + slotCount - 1) % slotCount;
@@ -166,7 +181,7 @@ class GlanceboardHomeLayer implements Layer {
* The board's slots scaled into `rect`: dividers (full brightness, or the
* dimmest dotted line when "Show lines" is off), the widget name centred in
* each region (a merged double-height region has one name and no line
* across it), and the selected quadrant highlighted.
* across it), and the selected slot highlighted.
*/
function drawSlotPreview(
image: GrayImage,
@@ -174,7 +189,7 @@ function drawSlotPreview(
options: { selectedSlot: number; selectionFocused: boolean },
): void {
const font = getDefaultSmallFont();
const layout = QUADRANT_LAYOUT;
const layout = glanceLayout();
const settings = glanceSlotSettings(layout);
const choices = settings.map((setting) => setting.get());
const regions = resolveGlanceRegions(layout, choices, glanceWidgetSpans);
@@ -232,14 +247,15 @@ function drawDottedLine(image: GrayImage, x0: number, y0: number, x1: number, y1
/** The picker for one slot: every widget, the current one marked; choosing applies the one-per-board rule. */
function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
const setting = glanceSlotSettings(QUADRANT_LAYOUT)[slotIndex];
const layout = glanceLayout();
const setting = glanceSlotSettings(layout)[slotIndex];
if (!setting) return;
const current = setting.get();
const items: MenuItem[] = setting.values.map((value) => ({
label: setting.displayValue(value),
onSelect: (menuCtx) => {
setting.set(value);
clearConflictingSlots(QUADRANT_LAYOUT, slotIndex, value);
clearConflictingSlots(layout, slotIndex, value);
menuCtx.stack.pop();
},
render: ({ image, x, y }) => {
@@ -249,7 +265,7 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current)));
}
/** Settings: which sleep gestures show the board, for how long, and the slot lines. */
/** Settings: layout, sleep gestures, duration, and slot lines. */
class GlanceSettingsLayer implements Layer {
private menu: MenuLayer | null = null;
@@ -257,6 +273,7 @@ class GlanceSettingsLayer implements Layer {
if (!this.menu) {
const { width } = ctx.stack.getBaseSize();
const items: MenuItem[] = [
enumSettingMenuItem(glanceLayoutSetting),
enumSettingMenuItem(glanceTapDurationSetting),
toggleSettingMenuItem(glanceShowOnLongPressSetting),
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
@@ -280,7 +297,7 @@ class GlanceSettingsLayer implements Layer {
class GlancePreviewLayer implements Layer {
private readonly board: GlanceBoard;
constructor(requestRender: () => void) {
constructor(requestRender: () => void, private readonly onClose: () => void) {
this.board = new GlanceBoard(requestRender);
this.board.start();
}
@@ -302,27 +319,40 @@ class GlancePreviewLayer implements Layer {
onRemoved(): void {
this.board.stop();
this.onClose();
}
}
export function createGlanceboardAppWindow(options: InProcessAppOptions): InProcessWindow {
return createInProcessWindow({
let previewVisible = false;
const heightMode = () => previewVisible && glanceLayoutSetting.get() === "2x3" ? "max" as const : "medium" as const;
let unsubscribeSettings: (() => void) | undefined;
const app = createInProcessWindow({
appId: "glanceboard",
windowId: GLANCEBOARD_WINDOW_ID,
title: "Glanceboard",
iconLetter: "Gb",
icon: "eye",
closeable: true,
// 576x288: the board's own size, so the preview is pixel-exact.
heightMode: "medium",
// Settings and the contents picker stay in the default 576x288 viewport;
// only the live six-slot preview needs the taller window.
heightMode: heightMode(),
actions: options.actions,
// Not wrapped in YieldAtRootLayer: the home page routes double-click
// itself (preview focus -> menu focus, menu focus -> sidebar).
baseLayer: new GlanceboardHomeLayer(),
baseLayer: new GlanceboardHomeLayer((visible) => {
previewVisible = visible;
app.setHeightMode(heightMode());
}),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
reconfigureSurface: options.reconfigureSurface,
onClosed: options.onClosed,
onClosed: () => {
unsubscribeSettings?.();
options.onClosed();
},
});
unsubscribeSettings = onAnySettingChanged(() => app.setHeightMode(heightMode()));
return app;
}
+27 -8
View File
@@ -1,5 +1,5 @@
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
import { QUADRANT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout";
import { QUADRANT_LAYOUT, SIX_SLOT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout";
import { type GlanceWidgetId } from "./widget";
import { glanceWidgetSpans } from "./widgets";
@@ -28,6 +28,19 @@ export function glanceSlotChoiceLabel(choice: GlanceSlotChoice): string {
*/
const DEFAULT_QUADRANT_CHOICES: readonly GlanceSlotChoice[] = ["system-card", "calendar", "music", "calendar"];
export const glanceLayoutSetting = new ConfigSettingEnum<"2x2" | "2x3">({
id: "glanceboard-layout",
label: "Layout",
storageKey: "glanceboard.layout",
defaultValue: "2x2",
values: ["2x2", "2x3"],
description: "Choose two columns with two or three rows of widgets.",
});
export function glanceLayout(): GlanceLayout {
return glanceLayoutSetting.get() === "2x3" ? SIX_SLOT_LAYOUT : QUADRANT_LAYOUT;
}
/**
* Whether sleep-time gestures show the board at all. Off restores the plain
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
@@ -79,7 +92,7 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
label: "Show on long press",
storageKey: "glanceboard.showOnLongPress",
defaultValue: true,
description: "Holding a long-press while asleep shows the Glanceboard until the press is released.",
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
});
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
@@ -101,17 +114,19 @@ export const glanceShowLinesSetting = new ConfigSettingBoolean({
});
/**
* One picker per layout slot, keyed by layout id and slot index so a future
* layout gets its own stored choices rather than inheriting the quadrants'.
* Both grid sizes share the original storage keys so changing row count
* preserves the first four choices and remembers the hidden bottom row.
*/
export function glanceSlotSettings(layout: GlanceLayout = QUADRANT_LAYOUT): ConfigSettingEnum<GlanceSlotChoice>[] {
export function glanceSlotSettings(layout: GlanceLayout = glanceLayout()): ConfigSettingEnum<GlanceSlotChoice>[] {
const grid = layout === QUADRANT_LAYOUT || layout === SIX_SLOT_LAYOUT;
const storageId = grid ? QUADRANT_LAYOUT.id : layout.id;
return layout.slots.map(
(slot, index) =>
new ConfigSettingEnum<GlanceSlotChoice>({
id: `glanceboard-${layout.id}-slot-${index}`,
label: slot.label,
storageKey: `glanceboard.${layout.id}.slot.${index}`,
defaultValue: (layout === QUADRANT_LAYOUT ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none",
storageKey: `glanceboard.${storageId}.slot.${index}`,
defaultValue: (grid ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none",
values: SLOT_CHOICES,
formatValue: glanceSlotChoiceLabel,
description: `Which widget fills the ${slot.label.toLowerCase()} slot of the Glanceboard.`,
@@ -130,9 +145,13 @@ export function clearConflictingSlots(layout: GlanceLayout, changedIndex: number
const settings = glanceSlotSettings(layout);
const spans = glanceWidgetSpans(choice);
const cleared: number[] = [];
let keptPartner = false;
settings.forEach((setting, index) => {
if (index === changedIndex || setting.get() !== choice) return;
if (spans && slotsVerticallyAdjacent(layout, changedIndex, index)) return;
if (spans && !keptPartner && slotsVerticallyAdjacent(layout, changedIndex, index)) {
keptPartner = true;
return;
}
setting.set("none");
cleared.push(index);
});
+6 -3
View File
@@ -3,12 +3,12 @@ import { GlanceBoard } from "./board";
import { createGlanceboardAppWindow, GLANCEBOARD_SURFACE_ID, GLANCEBOARD_WINDOW_ID } from "./glanceboard-app";
import {
glanceboardEnabledSetting,
glanceLayout,
glanceShowOnHeadTiltSetting,
glanceShowOnLongPressSetting,
glanceShowOnTap,
glanceTapTimeoutMs,
} from "./glanceboard-settings";
import { QUADRANT_LAYOUT } from "./layout";
const glanceboardApp: AppDefinition = {
appId: "glanceboard",
@@ -16,13 +16,16 @@ const glanceboardApp: AppDefinition = {
icon: "eye",
launch: (ctx) => ctx.launchInProcessApp(GLANCEBOARD_WINDOW_ID, GLANCEBOARD_SURFACE_ID, createGlanceboardAppWindow),
glanceboard: {
size: { width: QUADRANT_LAYOUT.width, height: QUADRANT_LAYOUT.height },
get size() {
const layout = glanceLayout();
return { width: layout.width, height: layout.height };
},
isEnabled: () => glanceboardEnabledSetting.get(),
showOnTap: glanceShowOnTap,
tapTimeoutMs: glanceTapTimeoutMs,
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
createBoard: (requestRender) => new GlanceBoard(requestRender, QUADRANT_LAYOUT),
createBoard: (requestRender) => new GlanceBoard(requestRender),
},
};
+14
View File
@@ -29,6 +29,20 @@ export const QUADRANT_LAYOUT: GlanceLayout = {
],
};
/** Six 288x144 slots: the same first two rows, plus a third at the bottom. */
export const SIX_SLOT_LAYOUT: GlanceLayout = {
id: "six-slots",
width: GLANCEBOARD_WIDTH,
height: 432,
slots: [
...QUADRANT_LAYOUT.slots.slice(0, 2),
{ label: "Middle left", rect: { x: 0, y: 144, width: 288, height: 144 } },
{ label: "Middle right", rect: { x: 288, y: 144, width: 288, height: 144 } },
{ label: "Bottom left", rect: { x: 0, y: 288, width: 288, height: 144 } },
{ label: "Bottom right", rect: { x: 288, y: 288, width: 288, height: 144 } },
],
};
/** Whether two slots stack vertically with matching width (one region if merged). */
export function slotsVerticallyAdjacent(layout: GlanceLayout, a: number, b: number): boolean {
const first = layout.slots[a]?.rect;
+4 -1
View File
@@ -730,7 +730,10 @@ class MicSession {
},
});
engine.setListener(this.captionListenerProxy);
engine.setAsrModelDir(isAsrModelReady() ? moonshineModelDir() : null);
// Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine)
// that only ever uses Moonshine; unaffected by the two-model on-device
// Whisper addition in native/voice-control.ts's push-to-talk path.
engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null);
engine.setSpeakerModelPath(micModelPath("speaker-embedding"));
engine.start();
this.captionEngine = engine;
+6 -3
View File
@@ -328,17 +328,20 @@ function voiceSpeakersMenu(): MenuLayer {
() => startMicModelDownload("speaker-embedding"),
),
modelItem(
// Live captions always use Moonshine (see the matching note in
// mic-session.ts); the second on-device model (Whisper) is a
// push-to-talk dictation option only, managed in Settings > Voice.
"Caption ASR model",
isAsrModelReady,
() => isAsrModelReady("moonshine"),
() => {
const state = asrModelState();
const state = asrModelState("moonshine");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (141 MB)";
},
startAsrModelDownload,
() => startAsrModelDownload("moonshine"),
),
modelItem(
"Re-diarization model",
+2 -2
View File
@@ -1912,9 +1912,9 @@ class DashboardController {
return this.beginVoiceCapture("ptt", endpointing);
}
private stopVoiceCapture(): void {
private stopVoiceCapture(): Promise<void> | void {
++this.pttCaptureGeneration;
voiceControlBridge.stopPushToTalk();
return voiceControlBridge.stopPushToTalk();
}
private startContinuousVoiceCapture(): void {
+1 -1
View File
@@ -77,7 +77,7 @@ export class GlanceHost {
const provider = this.options.getProvider();
if (!provider?.isEnabled()) return null;
if (gesture === "click" && !provider.showOnTap()) return null;
if (gesture === "long-press" && !provider.showOnLongPress()) return null;
if ((gesture === "long-press" || gesture === "short-then-long-press") && !provider.showOnLongPress()) return null;
if (gesture === "head-tilt" && !provider.showOnHeadTilt()) return null;
return glanceEventForGesture(gesture, this.isVisible());
}
+4 -4
View File
@@ -2,7 +2,7 @@
* The Glanceboard's visibility rules, as a pure reducer so they can be tested
* without the display plumbing. The board is an alternate sleep-time display:
* while the shell is asleep a press (single tap, or a head-tilt wake) shows it
* for a short while, a hold (long-press) shows it until the press is released,
* for a short while, a hold (long-press or short-then-long-press) shows it until release,
* and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds
* these events in and applies the resulting visibility to the compositor.
*
@@ -49,8 +49,7 @@ export function reduceGlance(
case "hold":
return { visible: true, holding: true, hideAtMs: null };
case "release":
// Releases also follow presses the board was never holding for (a
// tap-then-hold, a release delivered late); those mean nothing here.
// Releases delivered late or after a disabled hold gesture mean nothing here.
return state.holding ? GLANCE_HIDDEN : state;
case "timeout":
if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state;
@@ -67,7 +66,7 @@ export function reduceGlance(
* already handles or ignores while asleep.
*/
export function glanceEventForGesture(
gesture: "click" | "double-click" | "long-press" | "long-press-release" | "head-tilt" | string,
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | string,
boardVisible: boolean,
): GlanceEvent | null {
switch (gesture) {
@@ -75,6 +74,7 @@ export function glanceEventForGesture(
case "head-tilt":
return { type: "press" };
case "long-press":
case "short-then-long-press":
return { type: "hold" };
case "long-press-release":
return { type: "release" };
+172 -95
View File
@@ -4,50 +4,107 @@ declare const com: any;
declare const java: any;
/**
* Download management for the on-device transcription model (Moonshine, via
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
* demand into the same filesDir location that earlier releases copied the
* bundled files to, so upgraded installs that already used on-device
* transcription need no download. FaceclawVoiceController reads the files
* from this directory.
* Download management for the on-device transcription models (sherpa-onnx
* offline recognizers). Models are fetched on demand into filesDir rather
* than bundled in the APK; FaceclawVoiceController reads the files from the
* directory each model's `dirName` names.
*
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
* three files rather than one; they download sequentially through
* Two models, same shape:
* - "moonshine": the original on-device option (three files). The model is
* no longer bundled in the APK; it is fetched into the same filesDir
* location that earlier releases copied the bundled files to, so
* upgraded installs that already used on-device transcription need no
* download.
* - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en,
* int8-quantized. Picked over tiny.en for materially better accuracy
* (Whisper's own tiny/base WER gap is real and well documented) on the
* reasoning that FaceclawVoiceController.java no longer re-decodes a
* Whisper segment on every live-partial tick — see its ASR_WHISPER_*
* comment — so base.en's extra compute is a one-time cost per utterance,
* not a cost paid ~1.4x/second while the user is speaking. tiny.en is an
* easy swap (this file's numbers only) if base.en proves too slow on a
* real phone; nobody has measured that yet. Files are csukuangfj's own
* (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx
* release asset, matching the mirror-not-the-GH-tarball approach already
* used for Moonshine below. Every hash here was computed locally from a
* freshly downloaded copy of the file, not copied from a doc page.
*
* Mirrors the on-phone assistant model flow in llama.ts, except each model
* here is multiple files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file).
*/
export type AsrModelId = "moonshine" | "whisper-base-en";
type AsrModelFile = {
name: string;
sha256: string;
sizeBytes: number;
};
export const ASR_MODEL = {
label: "Moonshine (English)",
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
// this mirror serves the same files (sha256-verified) individually.
baseUrl:
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
files: [
{
name: "decoder_model_merged.ort",
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
sizeBytes: 109424400,
},
{
name: "encoder_model.ort",
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
sizeBytes: 31326816,
},
{
name: "tokens.txt",
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
sizeBytes: 549350,
},
] as AsrModelFile[],
totalBytes: 141300566,
type AsrModelDef = {
label: string;
dirName: string;
baseUrl: string;
files: AsrModelFile[];
totalBytes: number;
};
export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
moonshine: {
label: "Moonshine (English)",
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
// this mirror serves the same files (sha256-verified) individually.
baseUrl:
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
files: [
{
name: "decoder_model_merged.ort",
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
sizeBytes: 109424400,
},
{
name: "encoder_model.ort",
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
sizeBytes: 31326816,
},
{
name: "tokens.txt",
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
sizeBytes: 549350,
},
],
totalBytes: 141300566,
},
"whisper-base-en": {
label: "Whisper (English, base)",
dirName: "sherpa-onnx-whisper-base-en-int8",
// csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx
// maintainer's own mirror of the project's whisper export, individual
// files (no tar.bz2-unpack problem to begin with, but kept as the same
// per-file-with-pinned-hash shape as Moonshine above for consistency).
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/",
files: [
{
name: "base.en-encoder.int8.onnx",
sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823",
sizeBytes: 29120534,
},
{
name: "base.en-decoder.int8.onnx",
sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41",
sizeBytes: 130669978,
},
{
name: "base.en-tokens.txt",
sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930",
sizeBytes: 835554,
},
],
totalBytes: 160626066,
},
};
export type AsrModelState = {
@@ -56,121 +113,141 @@ export type AsrModelState = {
totalBytes: number;
};
let downloader: any = null;
// Bytes of files already fully downloaded in this run, plus progress within
// the file currently downloading; drives the aggregate percentage.
let completedBytes = 0;
let currentFileBytes = 0;
const stateListeners = new Set<(state: AsrModelState) => void>();
type ModelRuntime = {
downloader: any;
// Bytes of files already fully downloaded in this run, plus progress within
// the file currently downloading; drives the aggregate percentage.
completedBytes: number;
currentFileBytes: number;
stateListeners: Set<(state: AsrModelState) => void>;
};
function modelDirPath(): string {
function freshRuntime(): ModelRuntime {
return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() };
}
const runtimes: Record<AsrModelId, ModelRuntime> = {
moonshine: freshRuntime(),
"whisper-base-en": freshRuntime(),
};
function modelDirPath(id: AsrModelId): string {
const context = Utils.android.getApplicationContext();
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`;
}
function filePath(file: AsrModelFile): string {
return `${modelDirPath()}/${file.name}`;
function filePath(id: AsrModelId, file: AsrModelFile): string {
return `${modelDirPath(id)}/${file.name}`;
}
function isFilePresent(file: AsrModelFile): boolean {
function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean {
try {
const javaFile = new java.io.File(filePath(file));
const javaFile = new java.io.File(filePath(id, file));
return javaFile.exists() && javaFile.length() > 0;
} catch {
return false;
}
}
export function isAsrModelReady(): boolean {
export function isAsrModelReady(id: AsrModelId): boolean {
if (!global.isAndroid) return false;
return ASR_MODEL.files.every(isFilePresent);
return ASR_MODELS[id].files.every((file) => isFilePresent(id, file));
}
export function asrModelState(): AsrModelState {
if (downloader) {
export function asrModelState(id: AsrModelId): AsrModelState {
const runtime = runtimes[id];
const totalBytes = ASR_MODELS[id].totalBytes;
if (runtime.downloader) {
return {
status: "downloading",
bytesDownloaded: completedBytes + currentFileBytes,
totalBytes: ASR_MODEL.totalBytes,
bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes,
totalBytes,
};
}
return {
status: isAsrModelReady() ? "ready" : "absent",
status: isAsrModelReady(id) ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: ASR_MODEL.totalBytes,
totalBytes,
};
}
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void {
const runtime = runtimes[id];
runtime.stateListeners.add(listener);
return () => runtime.stateListeners.delete(listener);
}
function notifyStateChanged(): void {
const state = asrModelState();
stateListeners.forEach((listener) => listener(state));
function notifyStateChanged(id: AsrModelId): void {
const state = asrModelState(id);
runtimes[id].stateListeners.forEach((listener) => listener(state));
}
export function startAsrModelDownload(): void {
if (!global.isAndroid || downloader || isAsrModelReady()) return;
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
downloadNextFile();
notifyStateChanged();
export function startAsrModelDownload(id: AsrModelId): void {
const runtime = runtimes[id];
if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return;
runtime.completedBytes = ASR_MODELS[id].files
.filter((file) => isFilePresent(id, file))
.reduce((sum, f) => sum + f.sizeBytes, 0);
downloadNextFile(id);
notifyStateChanged(id);
}
function downloadNextFile(): void {
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
function downloadNextFile(id: AsrModelId): void {
const runtime = runtimes[id];
const def = ASR_MODELS[id];
const nextFile = def.files.find((file) => !isFilePresent(id, file));
if (!nextFile) {
downloader = null;
notifyStateChanged();
runtime.downloader = null;
notifyStateChanged(id);
return;
}
currentFileBytes = 0;
runtime.currentFileBytes = 0;
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
currentFileBytes = Number(bytes);
notifyStateChanged();
runtime.currentFileBytes = Number(bytes);
notifyStateChanged(id);
},
onDone: () => {
completedBytes += nextFile.sizeBytes;
currentFileBytes = 0;
downloadNextFile();
runtime.completedBytes += nextFile.sizeBytes;
runtime.currentFileBytes = 0;
downloadNextFile(id);
},
onError: (message: string) => {
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
downloader = null;
notifyStateChanged();
console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`);
runtime.downloader = null;
notifyStateChanged(id);
},
});
downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${ASR_MODEL.baseUrl}${nextFile.name}`,
filePath(nextFile),
runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${def.baseUrl}${nextFile.name}`,
filePath(id, nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloader.start();
runtime.downloader.start();
}
/** Stops the download; already-fetched bytes are kept and resumed next time. */
export function cancelAsrModelDownload(): void {
if (!downloader) return;
downloader.cancel();
downloader = null;
notifyStateChanged();
export function cancelAsrModelDownload(id: AsrModelId): void {
const runtime = runtimes[id];
if (!runtime.downloader) return;
runtime.downloader.cancel();
runtime.downloader = null;
notifyStateChanged(id);
}
export function deleteAsrModel(): void {
export function deleteAsrModel(id: AsrModelId): void {
if (!global.isAndroid) return;
cancelAsrModelDownload();
cancelAsrModelDownload(id);
try {
for (const file of ASR_MODEL.files) {
new java.io.File(filePath(file)).delete();
new java.io.File(`${filePath(file)}.part`).delete();
for (const file of ASR_MODELS[id].files) {
new java.io.File(filePath(id, file)).delete();
new java.io.File(`${filePath(id, file)}.part`).delete();
}
new java.io.File(modelDirPath()).delete();
new java.io.File(modelDirPath(id)).delete();
} catch (error) {
console.error(`Voice model delete failed: ${String(error)}`);
console.error(`Voice model delete failed (${id}): ${String(error)}`);
}
notifyStateChanged();
notifyStateChanged(id);
}
+26 -4
View File
@@ -26,7 +26,12 @@ export type VoiceControlState = {
detail: string;
};
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
// renamed, since it's a persisted settings value on real installs already.
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
@@ -103,6 +108,9 @@ export class FaceclawVoiceControlBridge {
private listening = false;
private detail = "";
private started = false;
private nativeCaptureFinished: Promise<void> = Promise.resolve();
private nextNativeCaptureId = 1;
private readonly nativeCaptureResolvers = new Map<number, () => void>();
// The mic is a single shared stream; these are the reasons it is running.
// The first holder starts capture (choosing the provider); the mic stops
// when the last one releases. Transcript events broadcast to every
@@ -184,8 +192,13 @@ export class FaceclawVoiceControlBridge {
}
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
stopPushToTalk(): void {
stopPushToTalk(): Promise<void> | void {
// Shared continuous capture keeps running; cloud providers finalize via
// their own callbacks. Only a stopped native recognizer must be awaited.
const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1
? this.nativeCaptureFinished : undefined;
this.releaseCapture("ptt", true);
return finished;
}
/** Begin continuous capture (Transcribe): the mic stays on until released. */
@@ -314,7 +327,12 @@ export class FaceclawVoiceControlBridge {
this.cloudClient = null;
this.started = true;
this.controller?.start("onboard");
// Which on-device model to load; a no-op setter for every provider except
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
const captureId = this.nextNativeCaptureId++;
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
this.controller?.start("onboard", captureId);
}
/**
@@ -323,7 +341,7 @@ export class FaceclawVoiceControlBridge {
* than failing the capture outright.
*/
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
if (options.provider === "onboard") return null;
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
const sttOptions = {
apiKey: "",
onTranscript: (event: CloudSttTranscriptEvent) =>
@@ -483,6 +501,10 @@ export class FaceclawVoiceControlBridge {
onTranscript: (text: string, isFinal: boolean) => {
this.emitTranscript(String(text), Boolean(isFinal));
},
onStopped: (captureId: number) => {
this.nativeCaptureResolvers.get(captureId)?.();
this.nativeCaptureResolvers.delete(captureId);
},
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
+12 -5
View File
@@ -523,12 +523,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
"How R1 ring input reaches the phone. Only via glasses: the ring's own link to the glasses carries its gestures, and the phone never opens a Bluetooth connection to the ring. Direct: also connect to the ring from the phone (currently unreliable). Takes effect on the next connection to the glasses.",
});
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
// "whisper" (no "onboard-" prefix) is OpenAI's CLOUD realtime model
// (gpt-realtime-whisper); "onboard-whisper" is the on-device sherpa-onnx
// Whisper backend. Same underlying model family, two different places it
// runs -- see the same note in native/voice-control.ts. The "whisper" value
// keeps its name (it's a persisted setting on real installs) but its label
// below now says "OpenAI" to tell the two apart in the picker.
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
const voiceProviderLabels: Record<VoiceProvider, string> = {
onboard: "On-device",
onboard: "On-device (Moonshine)",
"onboard-whisper": "On-device (Whisper)",
elevenlabs: "ElevenLabs",
whisper: "Whisper",
whisper: "OpenAI (Whisper)",
soniox: "Soniox",
};
@@ -537,7 +544,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
label: "Transcription Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: ["onboard", "elevenlabs", "whisper", "soniox"],
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
formatValue: (value) => voiceProviderLabels[value] ?? value,
isDisabled: (value) => {
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
@@ -545,7 +552,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
return false;
},
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).",
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
});
const wakeWordActionLabels: Record<WakeWordAction, string> = {
+31 -25
View File
@@ -13,12 +13,13 @@ import {
startLocalModelDownload,
} from "../../native/llama";
import {
ASR_MODEL,
ASR_MODELS,
asrModelState,
cancelAsrModelDownload,
deleteAsrModel,
onAsrModelStateChanged,
startAsrModelDownload,
type AsrModelId,
} from "../../native/asr-model";
import { TextViewerLayer } from "../../apps/files/text-viewer";
import type { LayerContext } from "../layers";
@@ -119,7 +120,8 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
asrModelMenuItem(),
asrModelMenuItem("moonshine"),
asrModelMenuItem("whisper-base-en"),
],
},
{
@@ -344,48 +346,52 @@ function localModelMenuItem(): MenuItem {
};
}
const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`;
function asrModelMb(id: AsrModelId): string {
return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`;
}
let asrModelRenderUnsub: (() => void) | null = null;
const asrModelRenderUnsub: Partial<Record<AsrModelId, () => void>> = {};
function watchAsrModelDownload(ctx: LayerContext): void {
asrModelRenderUnsub?.();
asrModelRenderUnsub = onAsrModelStateChanged((state) => {
function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void {
asrModelRenderUnsub[id]?.();
asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => {
ctx.actions.requestRender();
if (state.status !== "downloading") {
asrModelRenderUnsub?.();
asrModelRenderUnsub = null;
asrModelRenderUnsub[id]?.();
asrModelRenderUnsub[id] = undefined;
}
});
}
function asrModelStatusText(): string {
const state = asrModelState();
function asrModelStatusText(id: AsrModelId): string {
const state = asrModelState(id);
if (state.status === "ready") return "downloaded";
if (state.status === "downloading") {
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
return `${pct}% of ${ASR_MODEL_MB}`;
return `${pct}% of ${asrModelMb(id)}`;
}
return "not downloaded";
}
/** Download/cancel/delete management for the on-device transcription model. */
function asrModelMenuItem(): MenuItem {
/** Download/cancel/delete management for one on-device transcription model. */
function asrModelMenuItem(id: AsrModelId): MenuItem {
const def = ASR_MODELS[id];
const rowLabel = `On-device model: ${def.label}`;
return {
label: "On-device voice model",
label: rowLabel,
description:
`${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` +
`${def.label} (${asrModelMb(id)} download). ` +
"Transcribes voice input on the phone itself, with no API key or cloud service. " +
"Required for the On-device transcription provider; the cloud providers work without it. " +
"Required for its matching Transcription Provider option; the other providers work without it. " +
"An interrupted download resumes where it left off.",
onSelect: (ctx) => {
const state = asrModelState();
const state = asrModelState(id);
const action: MenuItem =
state.status === "downloading"
? {
label: "Cancel download",
onSelect: (innerCtx) => {
cancelAsrModelDownload();
cancelAsrModelDownload(id);
innerCtx.stack.pop();
},
}
@@ -393,22 +399,22 @@ function asrModelMenuItem(): MenuItem {
? {
label: "Delete model",
onSelect: (innerCtx) => {
deleteAsrModel();
deleteAsrModel(id);
innerCtx.stack.pop();
},
}
: {
label: `Download (${ASR_MODEL_MB})`,
label: `Download (${asrModelMb(id)})`,
onSelect: (innerCtx) => {
startAsrModelDownload();
watchAsrModelDownload(innerCtx);
startAsrModelDownload(id);
watchAsrModelDownload(id, innerCtx);
innerCtx.stack.pop();
},
};
openModalMenu(ctx, "On-device voice model", [action], 0);
openModalMenu(ctx, rowLabel, [action], 0);
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText());
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id));
},
};
}
+1 -1
View File
@@ -28,7 +28,7 @@ export type LayerActions = {
* (hands-free); otherwise it runs until stopVoiceCapture.
*/
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
stopVoiceCapture: () => Promise<void> | void;
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
startContinuousVoiceCapture: () => Promise<void> | void;
+56 -16
View File
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
// After native recognition finishes, a cloud final can still trail in.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
@@ -80,6 +79,10 @@ export class VoiceInputLayer implements Layer {
private capturing = false;
/** Continuation stopped; waiting for the trailing final transcript. */
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private pendingFollowup = false;
private stoppingCapture = false;
private stopGeneration = 0;
private acceptingTranscript = true;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
@@ -155,21 +158,48 @@ export class VoiceInputLayer implements Layer {
}
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.endsWith("...")) {
// A progress status ("Starting microphone...", "Listening...") gives
// way to the menu prompt; an error (ending in ".") stays visible.
this.status = "Send, continue, or discard?";
}
this.stopCapture(() => {
if (this.pendingAutoSend) {
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
}
/** Native stop can outlast its synchronous join while Whisper is decoding. */
private stopCapture(onStopped: () => void): void {
const generation = ++this.stopGeneration;
this.stoppingCapture = true;
const afterStop = () => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
onStopped();
this.actions.requestRender();
};
const stopped = this.actions.stopVoiceCapture();
if (!stopped) {
afterStop();
return;
}
void stopped.then(afterStop, (error) => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
this.pendingAutoSend = false;
this.acceptingTranscript = false;
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
});
}
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
@@ -193,7 +223,7 @@ export class VoiceInputLayer implements Layer {
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasText = text.length > 0 && !this.stoppingCapture;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
@@ -201,6 +231,7 @@ export class VoiceInputLayer implements Layer {
label: target.label,
dim: !hasText,
onSelect: () => {
if (this.stoppingCapture) return;
this.dismiss();
if (hasText) target.onSend(text);
},
@@ -208,9 +239,9 @@ export class VoiceInputLayer implements Layer {
}
rows.push({
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
dim: !hasLlmKey || this.stoppingCapture,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
@@ -222,7 +253,7 @@ export class VoiceInputLayer implements Layer {
const inMenu = this.phase === "menu";
paintInputDialog(image, {
title: this.capturing && this.listening ? "Voice ●" : "Voice",
status: this.status,
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
text: this.displayText() || this.placeholderText(),
rows: inMenu ? this.menuRows() : [],
selectedRow: this.menuIndex,
@@ -284,6 +315,7 @@ export class VoiceInputLayer implements Layer {
/** Continue selected: keep the message aside and record a follow-up. */
private startContinuation(): void {
this.acceptingTranscript = true;
this.baseText = this.displayText().trim();
this.finalizedText = "";
this.liveText = "";
@@ -298,15 +330,18 @@ export class VoiceInputLayer implements Layer {
private endContinuationCapture(): void {
if (!this.capturing) return;
this.capturing = false;
void this.actions.stopVoiceCapture();
this.status = "Refining...";
this.pendingFollowup = true;
this.status = "Finishing transcription...";
this.stopCapture(() => {
if (this.pendingFollowup) {
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
// The provider's committed transcript arrives shortly after stop; refine
// when it does, or after a timeout with whatever partials we have.
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
private beginRefine(): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -343,9 +378,10 @@ export class VoiceInputLayer implements Layer {
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
private cancelContinuation(status: string): void {
this.acceptingTranscript = false;
if (this.capturing) {
this.capturing = false;
void this.actions.stopVoiceCapture();
this.stopCapture(() => {});
}
this.refineHandle?.cancel();
this.refineHandle = null;
@@ -353,6 +389,7 @@ export class VoiceInputLayer implements Layer {
}
private backToMenu(text: string, status: string): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -366,6 +403,9 @@ export class VoiceInputLayer implements Layer {
}
onRemoved(): void {
++this.stopGeneration;
this.acceptingTranscript = false;
this.pendingFollowup = false;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
@@ -428,14 +468,14 @@ export class VoiceInputLayer implements Layer {
private onTranscript(event: VoiceTranscriptEvent): void {
// The refine stream owns the text buffers once it starts; a transcript
// that trails in after that point is stale.
if (this.phase === "refining") return;
if (!this.acceptingTranscript || this.phase === "refining") return;
if (event.isFinal) {
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
this.liveText = "";
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
if (this.phase === "continuing" && this.pendingFollowup) {
// The follow-up finalized; no need to keep waiting.
this.beginRefine();
return;