forked from Blendman974/faceclaw
Merge branch 'main' into bluetooth-pairing-improvements
This commit is contained in:
commit
ee022cdc84
41 files changed
+482
-1329
No files matched your search
@@ -116,7 +116,6 @@ export class AccelerometerDemoLayer implements Layer {
|
||||
image.drawText(small, 22, 82, "accelerometer at all — experimental.", 120);
|
||||
}
|
||||
|
||||
image.drawText(small, 20, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
|
||||
const HEADER_HEIGHT = 30;
|
||||
const LIST_X = 20;
|
||||
const FOOTER_HEIGHT = 22;
|
||||
const FOOTER_HEIGHT = 0;
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -58,7 +58,6 @@ export class BuzzerDemoLayer implements Layer {
|
||||
image.drawText(font, LIST_X + 110, y + 1, effect.desc, selected ? 180 : 120);
|
||||
}
|
||||
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_SCROLL} select ${GESTURE_CLICK} play ${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,9 @@ const MENU_LAYOUT = {
|
||||
showBorder: false,
|
||||
minHeight: 0,
|
||||
maxHeight: appViewportSize("min").height - 16,
|
||||
// These menus are pages, not popups: "Debug tests" replaces the root menu
|
||||
// visually instead of letting the taller root show through beneath it.
|
||||
opaque: true,
|
||||
};
|
||||
|
||||
/** A row that opens a nested page: the label plus a right-edge ">". */
|
||||
|
||||
@@ -78,7 +78,7 @@ export class ResourceUsageLayer implements Layer {
|
||||
image.drawText(
|
||||
font,
|
||||
width - 16 - font.measureText(`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`),
|
||||
37,
|
||||
47 - font.lineHeight,
|
||||
`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`,
|
||||
155,
|
||||
);
|
||||
@@ -105,7 +105,7 @@ export class ResourceUsageLayer implements Layer {
|
||||
]);
|
||||
|
||||
image.drawText(font, 16, height - 16, `-60s`, 80);
|
||||
const nowLabel = `now ${GESTURE_DOUBLE_CLICK} back`;
|
||||
const nowLabel = `now`;
|
||||
image.drawText(font, width - 16 - font.measureText(nowLabel), height - 16, nowLabel, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
+45
-29
@@ -5,7 +5,6 @@ import { clamp } from "../../util/numeric-util";
|
||||
import {
|
||||
GESTURE_CLICK,
|
||||
GESTURE_DOUBLE_CLICK,
|
||||
GESTURE_SCROLL,
|
||||
GESTURE_SCROLL_DOWN,
|
||||
GESTURE_SCROLL_UP,
|
||||
} from "../../ui/gestures";
|
||||
@@ -29,16 +28,19 @@ import {
|
||||
export const MUSIC_WINDOW_ID = "music";
|
||||
export const MUSIC_SURFACE_ID = "window:music";
|
||||
|
||||
const ART_SIZE = 96;
|
||||
const ART_X = 22;
|
||||
const ART_SIZE = 112;
|
||||
const ART_X = 8; // left margin matches the top margin
|
||||
const ART_Y = 8;
|
||||
const META_X = ART_X + ART_SIZE + 14;
|
||||
const LIST_TOP = 120;
|
||||
const PROGRESS_TIME_Y = ART_Y + 100;
|
||||
const PROGRESS_BAR_Y = ART_Y + 116;
|
||||
const LIST_TOP = 136;
|
||||
/** Gap between the menus' last row band and the viewport bottom. */
|
||||
const LIST_BOTTOM_MARGIN = 4;
|
||||
const ACTION_X = 22;
|
||||
const ACTION_WIDTH = 154;
|
||||
const COLUMN_DIVIDER_X = 190;
|
||||
const QUEUE_X = 204;
|
||||
const FOOTER_HEIGHT = 20;
|
||||
const PLAYLIST_ACTION_INDEX = 1;
|
||||
|
||||
type MusicAction =
|
||||
@@ -83,7 +85,7 @@ class MusicAppLayer implements Layer {
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
image.drawText(font, 24, 16 + index * lineStep(font), lines[index]!, 180);
|
||||
}
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings ${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings`, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -92,10 +94,9 @@ class MusicAppLayer implements Layer {
|
||||
if (mediaBrowserBridge.listBrowsableApps().length) {
|
||||
image.drawText(font, 24, 22 + lineStep(font), "Click to browse a music app's library,", 150);
|
||||
image.drawText(font, 24, 22 + 2 * lineStep(font), "or start playback on the phone.", 150);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} browse ${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} browse`, 110);
|
||||
} else {
|
||||
image.drawText(font, 24, 22 + lineStep(font), "Start playback in another app on the phone.", 150);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
@@ -104,21 +105,36 @@ class MusicAppLayer implements Layer {
|
||||
|
||||
const metaWidth = width - META_X - 24;
|
||||
const metaStep = lineStep(font) + 1;
|
||||
const titleLines = wrapText(font, media.title || "Unknown title", metaWidth).slice(0, 2);
|
||||
const metaTop = ART_Y + 2;
|
||||
// Line budget above the progress-bar time line: the layout is five slots
|
||||
// (title, second title line, artist, album, player app); when the font is
|
||||
// too large for all five, the title gets one truncated line instead of
|
||||
// wrapping to two, and the rest shift up a slot.
|
||||
const maxLines = Math.floor((PROGRESS_TIME_Y - metaTop - 4 - font.lineHeight) / metaStep) + 1;
|
||||
const titleSlots = maxLines >= 5 ? 2 : 1;
|
||||
const title = media.title || "Unknown title";
|
||||
const titleLines =
|
||||
titleSlots === 2 ? wrapText(font, title, metaWidth).slice(0, 2) : [truncateText(font, title, metaWidth)];
|
||||
for (let index = 0; index < titleLines.length; index++) {
|
||||
image.drawText(font, META_X, ART_Y + 2 + index * metaStep, titleLines[index]!, 230);
|
||||
image.drawText(font, META_X, metaTop + index * metaStep, titleLines[index]!, 230);
|
||||
}
|
||||
image.drawText(font, META_X, ART_Y + 6 + 2 * metaStep, media.artist || "Unknown artist", 180);
|
||||
image.drawText(font, META_X, metaTop + 4 + titleSlots * metaStep, media.artist || "Unknown artist", 180);
|
||||
if (media.album) {
|
||||
image.drawText(font, META_X, ART_Y + 6 + 3 * metaStep, truncateText(font, media.album, metaWidth), 150);
|
||||
image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150);
|
||||
}
|
||||
image.drawText(font, META_X, ART_Y + 6 + 4 * metaStep, truncateText(font, media.appName || media.packageName, metaWidth), 110);
|
||||
image.drawText(
|
||||
font,
|
||||
META_X,
|
||||
metaTop + 4 + (titleSlots + 2) * metaStep,
|
||||
truncateText(font, media.appName || media.packageName, metaWidth),
|
||||
110,
|
||||
);
|
||||
this.drawProgress(image, media, metaWidth);
|
||||
|
||||
const queue = mediaControllerBridge.getQueue();
|
||||
const actions = this.buildActions(media, queue);
|
||||
this.reconcileSelection(actions, queue);
|
||||
const listHeight = height - LIST_TOP - FOOTER_HEIGHT;
|
||||
const listHeight = height - LIST_TOP - LIST_BOTTOM_MARGIN;
|
||||
const rowH = tightRowHeight(font);
|
||||
const visibleRows = Math.max(1, (listHeight / rowH) | 0);
|
||||
if (this.selectedQueueIndex < this.queueScrollRow) {
|
||||
@@ -154,7 +170,7 @@ class MusicAppLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
image.drawLine(COLUMN_DIVIDER_X, LIST_TOP - 3, COLUMN_DIVIDER_X, height - FOOTER_HEIGHT - 3, 45);
|
||||
image.drawLine(COLUMN_DIVIDER_X, LIST_TOP - 3, COLUMN_DIVIDER_X, height - LIST_BOTTOM_MARGIN - 3, 45);
|
||||
if (!queue.length) {
|
||||
image.drawText(font, QUEUE_X, LIST_TOP + 1, "Playlist unavailable", 90);
|
||||
} else {
|
||||
@@ -189,14 +205,6 @@ class MusicAppLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
const backTarget = this.focusColumn === "playlist" ? "actions" : "back";
|
||||
image.drawText(
|
||||
font,
|
||||
20,
|
||||
height - 16,
|
||||
`${GESTURE_SCROLL} select ${GESTURE_CLICK} activate ${GESTURE_DOUBLE_CLICK} ${backTarget}`,
|
||||
110,
|
||||
);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -333,13 +341,12 @@ class MusicAppLayer implements Layer {
|
||||
private drawProgress(image: GrayImage, media: MediaControllerState, width: number): void {
|
||||
if (media.durationMs <= 0 || media.positionMs < 0) return;
|
||||
const font = getDefaultSmallFont();
|
||||
const y = ART_Y + 84;
|
||||
const elapsed = formatMediaTime(media.positionMs);
|
||||
const duration = formatMediaTime(media.durationMs);
|
||||
image.drawText(font, META_X, y, elapsed, 140);
|
||||
image.drawText(font, META_X + width - font.measureText(duration), y, duration, 140);
|
||||
image.drawText(font, META_X, PROGRESS_TIME_Y, elapsed, 140);
|
||||
image.drawText(font, META_X + width - font.measureText(duration), PROGRESS_TIME_Y, duration, 140);
|
||||
|
||||
const barY = ART_Y + 100;
|
||||
const barY = PROGRESS_BAR_Y;
|
||||
image.drawRect(META_X, barY, width, 5, 55);
|
||||
const progress = clamp(media.positionMs / media.durationMs, 0, 1);
|
||||
image.fillRect(META_X + 1, barY + 1, Math.round((width - 2) * progress), 3, 170);
|
||||
@@ -347,7 +354,10 @@ class MusicAppLayer implements Layer {
|
||||
|
||||
private drawArt(image: GrayImage, media: MediaControllerState): void {
|
||||
const key = `${media.packageName}|${media.title}|${media.album}`;
|
||||
if (key !== this.artKey) {
|
||||
// Retry while null: players (e.g. Spotify) often publish metadata before
|
||||
// the art bitmap is downloaded, then re-emit the same strings with the
|
||||
// bitmap attached. The no-art fetch path is a cheap near-no-op.
|
||||
if (key !== this.artKey || this.art === null) {
|
||||
this.artKey = key;
|
||||
this.art = mediaControllerBridge.getAlbumArt(ART_SIZE);
|
||||
}
|
||||
@@ -360,7 +370,13 @@ class MusicAppLayer implements Layer {
|
||||
} else {
|
||||
image.drawRect(ART_X, ART_Y, ART_SIZE, ART_SIZE, 60);
|
||||
const font = getDefaultSmallFont();
|
||||
image.drawText(font, ART_X + 22, ART_Y + ART_SIZE / 2 - 7, "no art", 90);
|
||||
image.drawText(
|
||||
font,
|
||||
ART_X + Math.round((ART_SIZE - font.measureText("no art")) / 2),
|
||||
ART_Y + Math.round((ART_SIZE - font.lineHeight) / 2),
|
||||
"no art",
|
||||
90,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,7 +195,6 @@ class DashboardController {
|
||||
private offFrameMetrics: (() => void) | null = null;
|
||||
private offFirmwareInfo: (() => void) | null = null;
|
||||
private offVoiceStatus: (() => void) | null = null;
|
||||
private offVoiceWakeWord: (() => void) | null = null;
|
||||
private offAndroidNotification: (() => void) | null = null;
|
||||
private lastInput = "waiting...";
|
||||
private lastSys = "none yet";
|
||||
@@ -1011,11 +1010,6 @@ class DashboardController {
|
||||
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.appendLog(state.status);
|
||||
});
|
||||
this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
|
||||
void this.handleWakeWord(keyword).catch((error) => {
|
||||
this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
|
||||
});
|
||||
});
|
||||
|
||||
await mediaControllerBridge.start();
|
||||
await nightscoutBridge.start();
|
||||
@@ -1082,8 +1076,6 @@ class DashboardController {
|
||||
this.offFirmwareInfo = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
@@ -1182,8 +1174,6 @@ class DashboardController {
|
||||
this.offFirmwareInfo = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
|
||||
const communicator = this.communicator;
|
||||
this.communicator = null;
|
||||
@@ -1816,20 +1806,6 @@ class DashboardController {
|
||||
this.updateCompositePreview();
|
||||
}
|
||||
|
||||
private async handleWakeWord(keyword: string): Promise<void> {
|
||||
const normalized = keyword.trim();
|
||||
if (normalized.length === 0) return;
|
||||
this.appendLog(`wake-word detected: ${normalized}`);
|
||||
const woke = shell.wake("sidebar");
|
||||
if (woke) {
|
||||
const ready = await this.ensureEvenHubSessionActive();
|
||||
if (!ready) {
|
||||
this.appendLog("EvenHub wake barrier timed out for phone wakeword");
|
||||
}
|
||||
this.requestShellRender();
|
||||
}
|
||||
}
|
||||
|
||||
private async handleAndroidNotificationPosted(notificationKey: string): Promise<void> {
|
||||
if (!notificationKey) {
|
||||
this.requestShellRender();
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Download management for the on-device transcription model (Moonshine, via
|
||||
* sherpa-onnx). The model is no longer bundled in the APK; it is fetched on
|
||||
* demand into the same filesDir location that earlier releases copied the
|
||||
* bundled files to, so upgraded installs that already used on-device
|
||||
* transcription need no download. FaceclawVoiceController reads the files
|
||||
* from this directory.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except the model is
|
||||
* three files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
export const ASR_MODEL = {
|
||||
label: "Moonshine (English)",
|
||||
dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27",
|
||||
// Hugging Face mirror of the sherpa-onnx release asset of the same name.
|
||||
// The GitHub release only offers a tar.bz2, which the phone can't unpack;
|
||||
// this mirror serves the same files (sha256-verified) individually.
|
||||
baseUrl:
|
||||
"https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/",
|
||||
files: [
|
||||
{
|
||||
name: "decoder_model_merged.ort",
|
||||
sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d",
|
||||
sizeBytes: 109424400,
|
||||
},
|
||||
{
|
||||
name: "encoder_model.ort",
|
||||
sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf",
|
||||
sizeBytes: 31326816,
|
||||
},
|
||||
{
|
||||
name: "tokens.txt",
|
||||
sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049",
|
||||
sizeBytes: 549350,
|
||||
},
|
||||
] as AsrModelFile[],
|
||||
totalBytes: 141300566,
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let downloader: any = null;
|
||||
// Bytes of files already fully downloaded in this run, plus progress within
|
||||
// the file currently downloading; drives the aggregate percentage.
|
||||
let completedBytes = 0;
|
||||
let currentFileBytes = 0;
|
||||
const stateListeners = new Set<(state: AsrModelState) => void>();
|
||||
|
||||
function modelDirPath(): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`;
|
||||
}
|
||||
|
||||
function filePath(file: AsrModelFile): string {
|
||||
return `${modelDirPath()}/${file.name}`;
|
||||
}
|
||||
|
||||
function isFilePresent(file: AsrModelFile): boolean {
|
||||
try {
|
||||
const javaFile = new java.io.File(filePath(file));
|
||||
return javaFile.exists() && javaFile.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAsrModelReady(): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
return ASR_MODEL.files.every(isFilePresent);
|
||||
}
|
||||
|
||||
export function asrModelState(): AsrModelState {
|
||||
if (downloader) {
|
||||
return {
|
||||
status: "downloading",
|
||||
bytesDownloaded: completedBytes + currentFileBytes,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: isAsrModelReady() ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: ASR_MODEL.totalBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = asrModelState();
|
||||
stateListeners.forEach((listener) => listener(state));
|
||||
}
|
||||
|
||||
export function startAsrModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isAsrModelReady()) return;
|
||||
completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0);
|
||||
downloadNextFile();
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
function downloadNextFile(): void {
|
||||
const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file));
|
||||
if (!nextFile) {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
return;
|
||||
}
|
||||
currentFileBytes = 0;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
currentFileBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
},
|
||||
onDone: () => {
|
||||
completedBytes += nextFile.sizeBytes;
|
||||
currentFileBytes = 0;
|
||||
downloadNextFile();
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Voice model download failed (${nextFile.name}): ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${ASR_MODEL.baseUrl}${nextFile.name}`,
|
||||
filePath(nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloader.start();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelAsrModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
export function deleteAsrModel(): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelAsrModelDownload();
|
||||
try {
|
||||
for (const file of ASR_MODEL.files) {
|
||||
new java.io.File(filePath(file)).delete();
|
||||
new java.io.File(`${filePath(file)}.part`).delete();
|
||||
}
|
||||
new java.io.File(modelDirPath()).delete();
|
||||
} catch (error) {
|
||||
console.error(`Voice model delete failed: ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
}
|
||||
@@ -45,7 +45,6 @@ export type RawPcmListener = (pcm: Uint8Array) => void;
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private readonly speechEndListeners = new Set<() => void>();
|
||||
private controller: any | null = null;
|
||||
@@ -72,11 +71,6 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.statusListeners.delete(listener);
|
||||
}
|
||||
|
||||
onWakeWord(listener: (keyword: string) => void): () => void {
|
||||
this.wakeWordListeners.add(listener);
|
||||
return () => this.wakeWordListeners.delete(listener);
|
||||
}
|
||||
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void {
|
||||
this.transcriptListeners.add(listener);
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
@@ -266,11 +260,6 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.rawActive) return;
|
||||
this.setStatus(String(status));
|
||||
},
|
||||
onWakeWord: (keyword: string) => {
|
||||
for (const listener of this.wakeWordListeners) {
|
||||
listener(String(keyword));
|
||||
}
|
||||
},
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
|
||||
@@ -319,12 +319,12 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription.",
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
"voice-input": "Voice Input",
|
||||
off: "Off",
|
||||
off: "Ignore",
|
||||
"turn-screen-on": "Turn Screen On",
|
||||
};
|
||||
|
||||
|
||||
@@ -12,6 +12,14 @@ import {
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
import {
|
||||
ASR_MODEL,
|
||||
asrModelState,
|
||||
cancelAsrModelDownload,
|
||||
deleteAsrModel,
|
||||
onAsrModelStateChanged,
|
||||
startAsrModelDownload,
|
||||
} from "../../native/asr-model";
|
||||
import { TextViewerLayer } from "../../apps/files/text-viewer";
|
||||
import type { LayerContext } from "../layers";
|
||||
import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu";
|
||||
@@ -87,6 +95,7 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
asrModelMenuItem(),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -240,6 +249,75 @@ function localModelMenuItem(): MenuItem {
|
||||
};
|
||||
}
|
||||
|
||||
const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`;
|
||||
|
||||
let asrModelRenderUnsub: (() => void) | null = null;
|
||||
|
||||
function watchAsrModelDownload(ctx: LayerContext): void {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = onAsrModelStateChanged((state) => {
|
||||
ctx.actions.requestRender();
|
||||
if (state.status !== "downloading") {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function asrModelStatusText(): string {
|
||||
const state = asrModelState();
|
||||
if (state.status === "ready") return "downloaded";
|
||||
if (state.status === "downloading") {
|
||||
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
|
||||
return `${pct}% of ${ASR_MODEL_MB}`;
|
||||
}
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-device transcription model. */
|
||||
function asrModelMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "On-device voice model",
|
||||
description:
|
||||
`${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` +
|
||||
"Transcribes voice input on the phone itself, with no API key or cloud service. " +
|
||||
"Required for the On-device transcription provider; the cloud providers work without it. " +
|
||||
"An interrupted download resumes where it left off.",
|
||||
onSelect: (ctx) => {
|
||||
const state = asrModelState();
|
||||
const action: MenuItem =
|
||||
state.status === "downloading"
|
||||
? {
|
||||
label: "Cancel download",
|
||||
onSelect: (innerCtx) => {
|
||||
cancelAsrModelDownload();
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: state.status === "ready"
|
||||
? {
|
||||
label: "Delete model",
|
||||
onSelect: (innerCtx) => {
|
||||
deleteAsrModel();
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: {
|
||||
label: `Download (${ASR_MODEL_MB})`,
|
||||
onSelect: (innerCtx) => {
|
||||
startAsrModelDownload();
|
||||
watchAsrModelDownload(innerCtx);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
};
|
||||
openModalMenu(ctx, "On-device voice model", [action], 0);
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A row that opens one of the project docs (copied into the bundle under
|
||||
* about/ by webpack.config.js) in the paged text viewer. */
|
||||
function bundledDocMenuItem(fileName: string, label: string): MenuItem {
|
||||
|
||||
@@ -142,7 +142,6 @@ export class FontPickerLayer implements Layer {
|
||||
const info = `Line height: ${preview.lineHeight}px`;
|
||||
image.drawText(font, rowX - 4, previewTop + 8 + preview.lineHeight + 6, info, 110);
|
||||
|
||||
image.drawText(font, rowX - 4, height - font.lineHeight - 8, `scroll select click change ${GESTURE_DOUBLE_CLICK} cancel`, 90);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -215,7 +214,7 @@ export class FontPickerLayer implements Layer {
|
||||
this.draft.kind === "bitmap"
|
||||
? Math.max(0, this.options.bitmapFaces.findIndex((b) => b.face === (this.draft as { face: BitmapFace }).face))
|
||||
: this.options.bitmapFaces.length + this.families.indexOf(this.draft.family);
|
||||
openModalMenu(ctx, "Font", items, Math.max(0, currentIndex));
|
||||
openModalMenu(ctx, "Font Face", items, Math.max(0, currentIndex));
|
||||
}
|
||||
|
||||
/** Switch families, carrying the weight (nearest available) and size over. */
|
||||
|
||||
+9
-2
@@ -26,6 +26,12 @@ export type MenuLayout = {
|
||||
minHeight?: number;
|
||||
/** Height cap before the menu starts scrolling. Default: the full screen. */
|
||||
maxHeight?: number;
|
||||
/**
|
||||
* Paint as a standalone page: the layers below stay in the stack for back
|
||||
* navigation but are not composited underneath. Default: false — the menu
|
||||
* renders over whatever is below it (modal/window menus).
|
||||
*/
|
||||
opaque?: boolean;
|
||||
};
|
||||
|
||||
export type MenuItemRenderArgs = {
|
||||
@@ -192,7 +198,8 @@ export class MenuLayer implements Layer {
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const rowHeight = listRowHeight(font);
|
||||
const image = paintBelow();
|
||||
const base = ctx.stack.getBaseSize();
|
||||
const image = this.layout.opaque ? new GrayImage(base.width, base.height, 0) : paintBelow();
|
||||
const { x, y, width } = this.layout;
|
||||
const chromeTop = (this.title ? menuTitleHeight(font) : 0) + MENU_BODY_PADDING;
|
||||
const minHeight = this.layout.minHeight ?? DEFAULT_MENU_MIN_HEIGHT;
|
||||
@@ -333,7 +340,7 @@ export class TextPageLayer implements Layer {
|
||||
private readonly body: string,
|
||||
) {}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
paint(_ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
image.drawText(font, 18, 14, this.title, 220);
|
||||
|
||||
@@ -129,7 +129,7 @@ export class AssistantLayer implements Layer {
|
||||
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
|
||||
}
|
||||
} else {
|
||||
image.drawText(font, left, top + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
|
||||
image.drawText(font, left, top + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110 - font.lineHeight);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -268,7 +268,7 @@ export class VoiceInputLayer implements Layer {
|
||||
image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
|
||||
}
|
||||
} else {
|
||||
image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110);
|
||||
image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110-font.lineHeight);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
export const FACECLAW_VERSION = "0.4.1";
|
||||
export const FACECLAW_VERSION = "0.5";
|
||||
export const USER_AGENT = "Faceclaw/"+FACECLAW_VERSION;
|
||||
Reference in new issue
Block a user