forked from Blendman974/faceclaw
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a80d6697b5
|
||
|
|
0b9e4cb367
|
No files matched your search
@@ -25,18 +25,21 @@ internal object AndroidSpeechEngines {
|
|||||||
private const val FEATURE_DIM = 80
|
private const val FEATURE_DIM = 80
|
||||||
|
|
||||||
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
||||||
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
|
fun recognizerConfig(
|
||||||
|
modelDir: File, kind: VoiceModelKind,
|
||||||
|
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
|
||||||
|
): OfflineRecognizerConfig {
|
||||||
val modelConfig = OfflineModelConfig.builder()
|
val modelConfig = OfflineModelConfig.builder()
|
||||||
.setNumThreads(1)
|
.setNumThreads(1)
|
||||||
if (kind == VoiceModelKind.WHISPER) {
|
if (kind == VoiceModelKind.WHISPER) {
|
||||||
modelConfig
|
modelConfig
|
||||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||||
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
|
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
|
||||||
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
|
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
|
||||||
.setLanguage("en")
|
.setLanguage(whisper.language)
|
||||||
.setTask("transcribe")
|
.setTask("transcribe")
|
||||||
.build())
|
.build())
|
||||||
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
|
.setTokens(File(modelDir, whisper.tokens).absolutePath)
|
||||||
} else {
|
} else {
|
||||||
modelConfig
|
modelConfig
|
||||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||||
|
|||||||
@@ -32,17 +32,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
"decoder_model_merged.ort",
|
"decoder_model_merged.ort",
|
||||||
"tokens.txt"
|
"tokens.txt"
|
||||||
)
|
)
|
||||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
|
||||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
|
||||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
|
||||||
// Whisper re-encodes the whole buffer on every call, so the shared session
|
|
||||||
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
|
|
||||||
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
|
|
||||||
private val ASR_WHISPER_MODEL_FILES = arrayOf(
|
|
||||||
"base.en-encoder.int8.onnx",
|
|
||||||
"base.en-decoder.int8.onnx",
|
|
||||||
"base.en-tokens.txt"
|
|
||||||
)
|
|
||||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||||
// (endpointing, transcript pacing) is tuned for.
|
// (endpointing, transcript pacing) is tuned for.
|
||||||
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
||||||
@@ -84,6 +73,7 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
private val mainHandler = Handler(Looper.getMainLooper())
|
private val mainHandler = Handler(Looper.getMainLooper())
|
||||||
@Volatile
|
@Volatile
|
||||||
private var communicator: FaceclawBleCommunicator? = null
|
private var communicator: FaceclawBleCommunicator? = null
|
||||||
|
private var whisperModel = WhisperModelFiles.forLanguage(null)
|
||||||
|
|
||||||
private val host = object : VoiceCaptureHost {
|
private val host = object : VoiceCaptureHost {
|
||||||
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
||||||
@@ -122,7 +112,7 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
|
|
||||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
||||||
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
||||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
|
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
||||||
@@ -168,6 +158,11 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
session.setOnboardModelKind(kind)
|
session.setOnboardModelKind(kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
|
||||||
|
fun setOnboardLanguage(language: String?) {
|
||||||
|
whisperModel = WhisperModelFiles.forLanguage(language)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||||
@@ -263,8 +258,8 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
*/
|
*/
|
||||||
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
||||||
val whisper = kind == VoiceModelKind.WHISPER
|
val whisper = kind == VoiceModelKind.WHISPER
|
||||||
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
|
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
|
||||||
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
|
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
|
||||||
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
||||||
for (fileName in fileNames) {
|
for (fileName in fileNames) {
|
||||||
val file = File(modelDir, fileName)
|
val file = File(modelDir, fileName)
|
||||||
|
|||||||
@@ -8,11 +8,7 @@ import { IconGrid, ICON_GRID_BAND_INSET_X, ICON_GRID_HIGHLIGHT_INSET_Y, ICON_GRI
|
|||||||
import { Layer, LayerActions, LayerContext } from "../../ui/layers";
|
import { Layer, LayerActions, LayerContext } from "../../ui/layers";
|
||||||
import { MenuLayer, type MenuItem } from "../../ui/menu";
|
import { MenuLayer, type MenuItem } from "../../ui/menu";
|
||||||
import { WINDOW_MENU_LAYOUT } from "../../ui/window-menu";
|
import { WINDOW_MENU_LAYOUT } from "../../ui/window-menu";
|
||||||
import {
|
import { onAnySettingChanged } from "../../ui/dashboard-settings";
|
||||||
onAnySettingChanged, launcherLayoutSetting, enumSettingMenuItem,
|
|
||||||
ConfigSettingString, EditTextSettingLayer,
|
|
||||||
} from "../../ui/dashboard-settings";
|
|
||||||
import { getLauncherLayoutFingerprint, orderLauncherEntries, moveLauncherEntry } from "./launcher-layout";
|
|
||||||
import { createInProcessWindow } from "../../ui/shell/in-process-window";
|
import { createInProcessWindow } from "../../ui/shell/in-process-window";
|
||||||
import {
|
import {
|
||||||
getFolderAssignments,
|
getFolderAssignments,
|
||||||
@@ -20,7 +16,6 @@ import {
|
|||||||
getFolderStateFingerprint,
|
getFolderStateFingerprint,
|
||||||
setAppFolder,
|
setAppFolder,
|
||||||
unusedNewFolderName,
|
unusedNewFolderName,
|
||||||
getExplicitFolderNames, createFolder, renameFolder, folderNameError, disbandFolder,
|
|
||||||
} from "./launcher-folders";
|
} from "./launcher-folders";
|
||||||
import { shell, type ShellWindow } from "../../ui/shell/shell";
|
import { shell, type ShellWindow } from "../../ui/shell/shell";
|
||||||
|
|
||||||
@@ -71,14 +66,6 @@ type LauncherGridEntry =
|
|||||||
* and is re-read every paint, so assistant folder tools take effect without
|
* and is re-read every paint, so assistant folder tools take effect without
|
||||||
* the layer holding any copy of the state.
|
* the layer holding any copy of the state.
|
||||||
*/
|
*/
|
||||||
const entryKey = (entry: LauncherGridEntry) => entry.kind === "app" ? `app:${entry.appId}` : `folder:${entry.name}`;
|
|
||||||
|
|
||||||
/** Reuse the phone text editor; dismiss it if the glasses leave this page. */
|
|
||||||
class FolderNameEditorLayer extends EditTextSettingLayer {
|
|
||||||
constructor(setting: ConfigSettingString, private readonly actions: LayerActions) { super(setting); }
|
|
||||||
onRemoved(): void { void this.actions.endTextSettingEdit(); }
|
|
||||||
}
|
|
||||||
|
|
||||||
class LauncherGridLayer implements Layer {
|
class LauncherGridLayer implements Layer {
|
||||||
// Watch swipes are spatial: up/down move between rows, left/right between
|
// Watch swipes are spatial: up/down move between rows, left/right between
|
||||||
// columns. From the leftmost column, left keeps going out: it leaves the
|
// columns. From the leftmost column, left keeps going out: it leaves the
|
||||||
@@ -87,12 +74,6 @@ class LauncherGridLayer implements Layer {
|
|||||||
readonly acceptsDirectional = true;
|
readonly acceptsDirectional = true;
|
||||||
/** Folder whose contents the grid is showing, or null for the top grid. */
|
/** Folder whose contents the grid is showing, or null for the top grid. */
|
||||||
private currentFolder: string | null = null;
|
private currentFolder: string | null = null;
|
||||||
private editingFolder = false;
|
|
||||||
private moving: { key: string; label: string; folder: string | null } | null = null;
|
|
||||||
|
|
||||||
private sortEntries(entries: LauncherGridEntry[]): LauncherGridEntry[] {
|
|
||||||
return orderLauncherEntries(entries, entryKey, this.currentFolder, launcherLayoutSetting.get() === "custom");
|
|
||||||
}
|
|
||||||
private readonly grid = new IconGrid<LauncherGridEntry>({
|
private readonly grid = new IconGrid<LauncherGridEntry>({
|
||||||
items: () => this.entries(),
|
items: () => this.entries(),
|
||||||
describe: (entry) => ({
|
describe: (entry) => ({
|
||||||
@@ -123,23 +104,25 @@ class LauncherGridLayer implements Layer {
|
|||||||
const members = (getFolders().get(this.currentFolder) ?? [])
|
const members = (getFolders().get(this.currentFolder) ?? [])
|
||||||
.map((appId) => byAppId.get(appId))
|
.map((appId) => byAppId.get(appId))
|
||||||
.filter(Boolean) as LauncherAppEntry[];
|
.filter(Boolean) as LauncherAppEntry[];
|
||||||
if (members.length > 0 || getExplicitFolderNames().includes(this.currentFolder)) {
|
if (members.length > 0) {
|
||||||
return this.sortEntries(members.map((app): LauncherGridEntry => ({
|
return members
|
||||||
|
.map((app): LauncherGridEntry => ({
|
||||||
kind: "app",
|
kind: "app",
|
||||||
label: app.label,
|
label: app.label,
|
||||||
icon: app.icon,
|
icon: app.icon,
|
||||||
appId: app.appId,
|
appId: app.appId,
|
||||||
renderIcon: app.renderIcon,
|
renderIcon: app.renderIcon,
|
||||||
})));
|
}))
|
||||||
|
.sort((a, b) => a.label.localeCompare(b.label));
|
||||||
}
|
}
|
||||||
this.currentFolder = null;
|
this.currentFolder = null;
|
||||||
}
|
}
|
||||||
const assignments = getFolderAssignments();
|
const assignments = getFolderAssignments();
|
||||||
const entries: LauncherGridEntry[] = [];
|
const entries: LauncherGridEntry[] = [];
|
||||||
// Legacy folders need a known member; explicitly created folders may be empty.
|
// A folder cell appears only when it holds at least one known app; stale
|
||||||
const explicit = new Set(getExplicitFolderNames());
|
// assignments to removed apps are ignored.
|
||||||
for (const [name, members] of getFolders()) {
|
for (const [name, members] of getFolders()) {
|
||||||
if (explicit.has(name) || members.some((appId) => byAppId.has(appId))) {
|
if (members.some((appId) => byAppId.has(appId))) {
|
||||||
entries.push({ kind: "folder", label: name, name });
|
entries.push({ kind: "folder", label: name, name });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -154,7 +137,7 @@ class LauncherGridLayer implements Layer {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return this.sortEntries(entries);
|
return entries.sort((a, b) => a.label.localeCompare(b.label));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Leave the current folder, putting the selection back on its cell. */
|
/** Leave the current folder, putting the selection back on its cell. */
|
||||||
@@ -168,16 +151,6 @@ class LauncherGridLayer implements Layer {
|
|||||||
|
|
||||||
/** Enter a folder or launch an app; true when that left the grid (a launch). */
|
/** Enter a folder or launch an app; true when that left the grid (a launch). */
|
||||||
private async openEntry(entry: LauncherGridEntry): Promise<boolean> {
|
private async openEntry(entry: LauncherGridEntry): Promise<boolean> {
|
||||||
if (this.moving) {
|
|
||||||
const moving = this.moving;
|
|
||||||
this.moving = null;
|
|
||||||
if (launcherLayoutSetting.get() === "custom" && moving.folder === this.currentFolder) {
|
|
||||||
moveLauncherEntry(this.currentFolder, this.entries().map(entryKey), moving.key, entryKey(entry));
|
|
||||||
const index = this.entries().findIndex(item => entryKey(item) === moving.key);
|
|
||||||
if (index >= 0) this.grid.selectIndex(index);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (entry.kind === "folder") {
|
if (entry.kind === "folder") {
|
||||||
this.currentFolder = entry.name;
|
this.currentFolder = entry.name;
|
||||||
this.grid.resetSelection();
|
this.grid.resetSelection();
|
||||||
@@ -193,13 +166,6 @@ class LauncherGridLayer implements Layer {
|
|||||||
|
|
||||||
/** Leave the open folder, else yield to the sidebar; true when that left the grid. */
|
/** Leave the open folder, else yield to the sidebar; true when that left the grid. */
|
||||||
private back(): boolean {
|
private back(): boolean {
|
||||||
if (this.moving) {
|
|
||||||
const key = this.moving.key;
|
|
||||||
this.moving = null;
|
|
||||||
const index = this.entries().findIndex(entry => entryKey(entry) === key);
|
|
||||||
if (index >= 0) this.grid.selectIndex(index);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (this.currentFolder !== null) {
|
if (this.currentFolder !== null) {
|
||||||
this.exitFolder();
|
this.exitFolder();
|
||||||
return false;
|
return false;
|
||||||
@@ -209,54 +175,14 @@ class LauncherGridLayer implements Layer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Layout and folder creation are always available. Per-icon actions need
|
* App-specific entries for the window's context menu. Only item mode
|
||||||
* item mode so row navigation never edits an ambiguous selection.
|
* pins the selection to a single cell, and only apps can move to folders,
|
||||||
|
* so anywhere else the menu falls back to just the defaults.
|
||||||
*/
|
*/
|
||||||
menuItems(): MenuItem[] {
|
menuItems(): MenuItem[] {
|
||||||
if (this.editingFolder) return [];
|
|
||||||
const entry = this.grid.selectedItem;
|
const entry = this.grid.selectedItem;
|
||||||
const items: MenuItem[] = [
|
if (!entry || entry.kind !== "app") return [];
|
||||||
enumSettingMenuItem(launcherLayoutSetting, { onChange: () => { this.moving = null; this.grid.resetSelection(); } }),
|
return [
|
||||||
{
|
|
||||||
label: "New folder",
|
|
||||||
onSelect: ctx => {
|
|
||||||
const name = unusedNewFolderName();
|
|
||||||
if (createFolder(name)) {
|
|
||||||
this.currentFolder = null;
|
|
||||||
this.moving = null;
|
|
||||||
this.grid.resetSelection();
|
|
||||||
this.grid.selectIndex(this.entries().findIndex(item => item.kind === "folder" && item.name === name));
|
|
||||||
}
|
|
||||||
ctx.stack.pop();
|
|
||||||
},
|
|
||||||
},
|
|
||||||
];
|
|
||||||
const folder = entry?.kind === "folder" ? entry.name : this.currentFolder;
|
|
||||||
if (folder !== null) {
|
|
||||||
items.push({ label: "Rename folder", onSelect: ctx => this.editFolderName(folder, ctx) }, {
|
|
||||||
label: "Delete folder",
|
|
||||||
onSelect: ctx => {
|
|
||||||
ctx.stack.pop();
|
|
||||||
ctx.stack.push(new MenuLayer("Delete folder? Apps stay installed.", [
|
|
||||||
{ label: "Delete folder", onSelect: confirm => {
|
|
||||||
disbandFolder(folder);
|
|
||||||
if (this.currentFolder === folder) this.currentFolder = null;
|
|
||||||
this.moving = null;
|
|
||||||
this.grid.resetSelection();
|
|
||||||
confirm.stack.pop();
|
|
||||||
} },
|
|
||||||
{ label: "Cancel", onSelect: confirm => confirm.stack.pop() },
|
|
||||||
], WINDOW_MENU_LAYOUT));
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (this.moving) items.push({ label: "Cancel move", onSelect: ctx => { this.back(); ctx.stack.pop(); } });
|
|
||||||
else if (entry && launcherLayoutSetting.get() === "custom") items.push({
|
|
||||||
label: "Move icon",
|
|
||||||
onSelect: ctx => { this.moving = { key: entryKey(entry), label: entry.label, folder: this.currentFolder }; ctx.stack.pop(); },
|
|
||||||
});
|
|
||||||
if (!entry || entry.kind !== "app") return items;
|
|
||||||
return [...items,
|
|
||||||
{
|
{
|
||||||
label: "Move to folder",
|
label: "Move to folder",
|
||||||
onSelect: (ctx) => {
|
onSelect: (ctx) => {
|
||||||
@@ -295,32 +221,6 @@ class LauncherGridLayer implements Layer {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
private editFolderName(folder: string, ctx: LayerContext): void {
|
|
||||||
ctx.stack.pop();
|
|
||||||
const setting = new ConfigSettingString({
|
|
||||||
id: "launcher-folder-name", label: "Folder name", storageKey: "launcher.folderNameDraft",
|
|
||||||
defaultValue: folder, normalize: value => value ?? "",
|
|
||||||
validate: value => folderNameError(value, folder), reportInAnalytics: false,
|
|
||||||
});
|
|
||||||
setting.set(folder);
|
|
||||||
this.editingFolder = true;
|
|
||||||
const editor = new FolderNameEditorLayer(setting, ctx.actions);
|
|
||||||
ctx.stack.push(editor);
|
|
||||||
void ctx.actions.startTextSettingsEdit([setting], "Rename folder", () => {
|
|
||||||
this.editingFolder = false;
|
|
||||||
if (renameFolder(folder, setting.get())) {
|
|
||||||
if (this.currentFolder === folder) this.currentFolder = setting.get().trim();
|
|
||||||
this.moving = null;
|
|
||||||
}
|
|
||||||
ctx.stack.popIfTop(layer => layer === editor);
|
|
||||||
ctx.actions.requestRender();
|
|
||||||
}, undefined, () => {
|
|
||||||
this.editingFolder = false;
|
|
||||||
ctx.stack.popIfTop(layer => layer === editor);
|
|
||||||
ctx.actions.requestRender();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The folder-picker submenu: every existing folder except the app's own,
|
* The folder-picker submenu: every existing folder except the app's own,
|
||||||
* plus a generated-name "New folder" (ring input has no text entry) and,
|
* plus a generated-name "New folder" (ring input has no text entry) and,
|
||||||
@@ -342,9 +242,7 @@ class LauncherGridLayer implements Layer {
|
|||||||
items.push({
|
items.push({
|
||||||
label: "New folder",
|
label: "New folder",
|
||||||
onSelect: (ctx) => {
|
onSelect: (ctx) => {
|
||||||
const name = unusedNewFolderName();
|
setAppFolder(appId, unusedNewFolderName());
|
||||||
createFolder(name);
|
|
||||||
setAppFolder(appId, name);
|
|
||||||
ctx.stack.pop();
|
ctx.stack.pop();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -365,16 +263,8 @@ class LauncherGridLayer implements Layer {
|
|||||||
const { width, height } = ctx.stack.getBaseSize();
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
const image = new GrayImage(width, height, 0);
|
const image = new GrayImage(width, height, 0);
|
||||||
// The folder-name header band scales with the font.
|
// The folder-name header band scales with the font.
|
||||||
const entries = this.entries();
|
const gridTop = this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : ROOT_GRID_TOP;
|
||||||
if (this.moving && (launcherLayoutSetting.get() !== "custom" || this.moving.folder !== this.currentFolder ||
|
if (this.currentFolder !== null) {
|
||||||
!entries.some(entry => entryKey(entry) === this.moving!.key))) this.moving = null;
|
|
||||||
const gridTop = this.moving ? GRID_TOP + 2 * (font.lineHeight + 2)
|
|
||||||
: this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : ROOT_GRID_TOP;
|
|
||||||
if (this.moving) {
|
|
||||||
image.drawText(font, 8, GRID_TOP - 2, truncateText(font, `Move ${this.moving.label}`, width - 16), 220);
|
|
||||||
image.drawText(font, 8, GRID_TOP + font.lineHeight, "Select position; double tap cancels", 160);
|
|
||||||
}
|
|
||||||
if (this.currentFolder !== null && !this.moving) {
|
|
||||||
image.drawText(font, 8, GRID_TOP - 2, truncateText(font, this.currentFolder, width - 16), 160);
|
image.drawText(font, 8, GRID_TOP - 2, truncateText(font, this.currentFolder, width - 16), 160);
|
||||||
}
|
}
|
||||||
// The grid runs to the window's bottom edge, so a row cut off there for
|
// The grid runs to the window's bottom edge, so a row cut off there for
|
||||||
@@ -384,7 +274,6 @@ class LauncherGridLayer implements Layer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||||
if (this.moving && event.type === "double-click") { this.back(); return; }
|
|
||||||
await this.grid.handleInput(event, ctx);
|
await this.grid.handleInput(event, ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -422,9 +311,9 @@ export function createLauncherWindow(options: LauncherOptions): ShellWindow {
|
|||||||
// keeps every unrelated setting change from repainting the launcher. The
|
// keeps every unrelated setting change from repainting the launcher. The
|
||||||
// launcher is pinned for the app's lifetime, so the subscription never needs
|
// launcher is pinned for the app's lifetime, so the subscription never needs
|
||||||
// tearing down.
|
// tearing down.
|
||||||
let lastState = `${getFolderStateFingerprint()}\n${getLauncherLayoutFingerprint()}\n${launcherAppsFingerprint(options.apps())}`;
|
let lastState = `${getFolderStateFingerprint()}\n${launcherAppsFingerprint(options.apps())}`;
|
||||||
onAnySettingChanged(() => {
|
onAnySettingChanged(() => {
|
||||||
const state = `${getFolderStateFingerprint()}\n${getLauncherLayoutFingerprint()}\n${launcherAppsFingerprint(options.apps())}`;
|
const state = `${getFolderStateFingerprint()}\n${launcherAppsFingerprint(options.apps())}`;
|
||||||
if (state === lastState) return;
|
if (state === lastState) return;
|
||||||
lastState = state;
|
lastState = state;
|
||||||
created.requestRender();
|
created.requestRender();
|
||||||
|
|||||||
@@ -1,50 +1,14 @@
|
|||||||
/** Launcher assignments keep their legacy format; explicitly created folders may be empty. */
|
/**
|
||||||
|
* Launcher folder state: which apps are grouped into which named folder.
|
||||||
|
* Stored as one JSON string (appId -> folder name) in the settings store, so
|
||||||
|
* it persists and its changes broadcast to every isolate. An unset value means
|
||||||
|
* the default grouping; any edit persists the whole map, so clearing out the
|
||||||
|
* defaults sticks. Folders exist only while at least one app is assigned to
|
||||||
|
* them — there is no separate create/delete step.
|
||||||
|
*/
|
||||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||||
|
|
||||||
import { renameLauncherFolderOrder, removeLauncherFolderOrder } from "./launcher-layout";
|
|
||||||
|
|
||||||
const STORAGE_KEY = "launcher.folders";
|
const STORAGE_KEY = "launcher.folders";
|
||||||
const NAMES_KEY = "launcher.folderNames";
|
|
||||||
|
|
||||||
/** Only user-created/renamed folders persist after their last member leaves. */
|
|
||||||
export function getExplicitFolderNames(): string[] {
|
|
||||||
try {
|
|
||||||
const value = JSON.parse(getStringSetting(NAMES_KEY, "[]"));
|
|
||||||
return Array.isArray(value) ? [...new Set(value.filter((name): name is string =>
|
|
||||||
typeof name === "string" && !!name.trim()).map(name => name.trim()))] : [];
|
|
||||||
} catch { return []; }
|
|
||||||
}
|
|
||||||
|
|
||||||
export function folderNameError(input: string, previous?: string): string | null {
|
|
||||||
const name = input.trim();
|
|
||||||
if (!name) return "Enter a folder name.";
|
|
||||||
if (name.length > FOLDER_NAME_MAX_LENGTH) return `Use at most ${FOLDER_NAME_MAX_LENGTH} characters.`;
|
|
||||||
if (/[\u0000-\u001f\u007f]/.test(name)) return "Use a single-line folder name.";
|
|
||||||
if (previous !== undefined && !getFolders().has(previous)) return "This folder no longer exists.";
|
|
||||||
for (const existing of getFolders().keys()) {
|
|
||||||
if (existing !== previous && existing.toLowerCase() === name.toLowerCase()) return "A folder with this name already exists.";
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createFolder(input: string): boolean {
|
|
||||||
if (folderNameError(input)) return false;
|
|
||||||
setStringSetting(NAMES_KEY, JSON.stringify([...getExplicitFolderNames(), input.trim()]));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function renameFolder(previous: string, input: string): boolean {
|
|
||||||
if (folderNameError(input, previous)) return false;
|
|
||||||
const name = input.trim();
|
|
||||||
if (name === previous) return true;
|
|
||||||
const assignments = getFolderAssignments();
|
|
||||||
for (const appId of Object.keys(assignments)) if (assignments[appId] === previous) assignments[appId] = name;
|
|
||||||
// Preserve position and per-folder order when the visible name changes.
|
|
||||||
renameLauncherFolderOrder(previous, name);
|
|
||||||
setStringSetting(NAMES_KEY, JSON.stringify([...getExplicitFolderNames().filter(n => n !== previous), name]));
|
|
||||||
setStringSetting(STORAGE_KEY, JSON.stringify(assignments));
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The out-of-the-box grouping, applied only while the setting is unset. */
|
/** The out-of-the-box grouping, applied only while the setting is unset. */
|
||||||
const DEFAULT_ASSIGNMENTS: Record<string, string> = {
|
const DEFAULT_ASSIGNMENTS: Record<string, string> = {
|
||||||
@@ -68,7 +32,7 @@ export function getFolderAssignments(): Record<string, string> {
|
|||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(raw);
|
const parsed = JSON.parse(raw);
|
||||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
|
||||||
const assignments: Record<string, string> = Object.create(null);
|
const assignments: Record<string, string> = {};
|
||||||
for (const appId of Object.keys(parsed)) {
|
for (const appId of Object.keys(parsed)) {
|
||||||
const folder = parsed[appId];
|
const folder = parsed[appId];
|
||||||
if (typeof folder === "string" && folder.trim()) {
|
if (typeof folder === "string" && folder.trim()) {
|
||||||
@@ -94,7 +58,6 @@ export function getFolders(): Map<string, string[]> {
|
|||||||
unsorted.set(folder, [appId]);
|
unsorted.set(folder, [appId]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (const name of getExplicitFolderNames()) if (!unsorted.has(name)) unsorted.set(name, []);
|
|
||||||
const folders = new Map<string, string[]>();
|
const folders = new Map<string, string[]>();
|
||||||
for (const name of Array.from(unsorted.keys()).sort((a, b) => a.localeCompare(b))) {
|
for (const name of Array.from(unsorted.keys()).sort((a, b) => a.localeCompare(b))) {
|
||||||
folders.set(name, unsorted.get(name)!);
|
folders.set(name, unsorted.get(name)!);
|
||||||
@@ -123,8 +86,6 @@ export function disbandFolder(folder: string): number {
|
|||||||
moved++;
|
moved++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
setStringSetting(NAMES_KEY, JSON.stringify(getExplicitFolderNames().filter(name => name !== folder)));
|
|
||||||
removeLauncherFolderOrder(folder);
|
|
||||||
if (moved > 0) {
|
if (moved > 0) {
|
||||||
setStringSetting(STORAGE_KEY, JSON.stringify(assignments));
|
setStringSetting(STORAGE_KEY, JSON.stringify(assignments));
|
||||||
}
|
}
|
||||||
@@ -134,7 +95,7 @@ export function disbandFolder(folder: string): number {
|
|||||||
/**
|
/**
|
||||||
* An unused name for a ring-created folder ("New folder", "New folder 2", …).
|
* An unused name for a ring-created folder ("New folder", "New folder 2", …).
|
||||||
* Ring input has no text entry, so new folders start with a generated name;
|
* Ring input has no text entry, so new folders start with a generated name;
|
||||||
* Rename folder uses the existing phone text editor for a custom name.
|
* the assistant's folder tools can regroup the apps under a better one.
|
||||||
*/
|
*/
|
||||||
export function unusedNewFolderName(): string {
|
export function unusedNewFolderName(): string {
|
||||||
const existing = new Set(Array.from(getFolders().keys()).map((name) => name.toLowerCase()));
|
const existing = new Set(Array.from(getFolders().keys()).map((name) => name.toLowerCase()));
|
||||||
@@ -158,6 +119,5 @@ export function resolveFolderName(input: string): string {
|
|||||||
|
|
||||||
/** Raw stored value, for cheap change detection by the launcher window. */
|
/** Raw stored value, for cheap change detection by the launcher window. */
|
||||||
export function getFolderStateFingerprint(): string {
|
export function getFolderStateFingerprint(): string {
|
||||||
return JSON.stringify([getStringSetting(STORAGE_KEY, ""), getStringSetting(NAMES_KEY, ""),
|
return getStringSetting(STORAGE_KEY, "");
|
||||||
getStringSetting("launcher.folderNameDraft", "")]);
|
|
||||||
}
|
}
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
|
||||||
|
|
||||||
const ORDER_KEY = "launcher.order";
|
|
||||||
const scopeKey = (folder: string | null) => folder === null ? "root" : `folder:${folder}`;
|
|
||||||
|
|
||||||
function readOrders(): Record<string, string[]> {
|
|
||||||
try {
|
|
||||||
const value = JSON.parse(getStringSetting(ORDER_KEY, "{}"));
|
|
||||||
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
|
||||||
return Object.fromEntries(Object.entries(value).filter(([, order]) => Array.isArray(order))
|
|
||||||
.map(([scope, order]) => [scope, [...new Set((order as unknown[]).filter((id): id is string => typeof id === "string"))]]));
|
|
||||||
} catch { return {}; }
|
|
||||||
}
|
|
||||||
|
|
||||||
export function setLauncherOrder(folder: string | null, ids: readonly string[]): void {
|
|
||||||
const orders = readOrders();
|
|
||||||
orders[scopeKey(folder)] = [...new Set(ids)];
|
|
||||||
setStringSetting(ORDER_KEY, JSON.stringify(orders));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Unseen apps append alphabetically; changing sort modes never destroys saved order. */
|
|
||||||
export function orderLauncherEntries<T extends { label: string }>(
|
|
||||||
entries: T[], key: (entry: T) => string, folder: string | null, custom: boolean,
|
|
||||||
): T[] {
|
|
||||||
const order = custom ? readOrders()[scopeKey(folder)] ?? [] : [];
|
|
||||||
const rank = new Map(order.map((id, index) => [id, index]));
|
|
||||||
return entries.sort((a, b) => (rank.get(key(a)) ?? Infinity) - (rank.get(key(b)) ?? Infinity) ||
|
|
||||||
a.label.localeCompare(b.label) || key(a).localeCompare(key(b)));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Move into the target's position, without moving apps across folders. */
|
|
||||||
export function moveLauncherEntry(folder: string | null, ids: string[], source: string, target: string): boolean {
|
|
||||||
const from = ids.indexOf(source), to = ids.indexOf(target);
|
|
||||||
if (from < 0 || to < 0) return false;
|
|
||||||
const next = ids.slice();
|
|
||||||
next.splice(from, 1);
|
|
||||||
next.splice(to, 0, source);
|
|
||||||
setLauncherOrder(folder, next);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function renameLauncherFolderOrder(previous: string, next: string): void {
|
|
||||||
const orders = readOrders();
|
|
||||||
const oldScope = scopeKey(previous), newScope = scopeKey(next);
|
|
||||||
if (orders[oldScope]) orders[newScope] = orders[oldScope];
|
|
||||||
delete orders[oldScope];
|
|
||||||
if (orders.root) orders.root = orders.root.map(id => id === `folder:${previous}` ? `folder:${next}` : id);
|
|
||||||
setStringSetting(ORDER_KEY, JSON.stringify(orders));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function removeLauncherFolderOrder(folder: string): void {
|
|
||||||
const orders = readOrders();
|
|
||||||
delete orders[scopeKey(folder)];
|
|
||||||
if (orders.root) orders.root = orders.root.filter(id => id !== `folder:${folder}`);
|
|
||||||
setStringSetting(ORDER_KEY, JSON.stringify(orders));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getLauncherLayoutFingerprint(): string {
|
|
||||||
return JSON.stringify([getStringSetting("launcher.layout", "alphabetical"), getStringSetting(ORDER_KEY, "")]);
|
|
||||||
}
|
|
||||||
@@ -191,7 +191,7 @@ function registerTools(options: WindowToolsOptions, registry: ToolRegistry): voi
|
|||||||
const lines: string[] = [];
|
const lines: string[] = [];
|
||||||
for (const [name, members] of getFolders()) {
|
for (const [name, members] of getFolders()) {
|
||||||
const known = members.filter((appId) => appIds.indexOf(appId) >= 0).sort();
|
const known = members.filter((appId) => appIds.indexOf(appId) >= 0).sort();
|
||||||
if (known.length || members.length === 0) lines.push(`${name}: ${known.length ? known.map(describeApp).join(", ") : "(empty)"}`);
|
if (known.length) lines.push(`${name}: ${known.map(describeApp).join(", ")}`);
|
||||||
}
|
}
|
||||||
const assignments = getFolderAssignments();
|
const assignments = getFolderAssignments();
|
||||||
const ungrouped = appIds.filter((appId) => !assignments[appId]);
|
const ungrouped = appIds.filter((appId) => !assignments[appId]);
|
||||||
@@ -237,7 +237,7 @@ function registerTools(options: WindowToolsOptions, registry: ToolRegistry): voi
|
|||||||
{
|
{
|
||||||
name: "apps.remove_from_folder",
|
name: "apps.remove_from_folder",
|
||||||
description:
|
description:
|
||||||
"Move an app out of its launcher folder back to the top-level grid. Default folders disappear when their last app is removed; user-created folders can stay empty.",
|
"Move an app out of its launcher folder back to the top-level grid. A folder disappears when its last app is removed.",
|
||||||
inputSchema: {
|
inputSchema: {
|
||||||
type: "object",
|
type: "object",
|
||||||
properties: {
|
properties: {
|
||||||
@@ -276,9 +276,8 @@ function registerTools(options: WindowToolsOptions, registry: ToolRegistry): voi
|
|||||||
(args) => {
|
(args) => {
|
||||||
const folder = resolveFolderName(String(args?.folder ?? ""));
|
const folder = resolveFolderName(String(args?.folder ?? ""));
|
||||||
if (!folder) return err("folder must be a non-empty name");
|
if (!folder) return err("folder must be a non-empty name");
|
||||||
const existed = getFolders().has(folder);
|
|
||||||
const moved = disbandFolder(folder);
|
const moved = disbandFolder(folder);
|
||||||
if (moved === 0 && !existed) {
|
if (moved === 0) {
|
||||||
const names = Array.from(getFolders().keys());
|
const names = Array.from(getFolders().keys());
|
||||||
return err(
|
return err(
|
||||||
names.length
|
names.length
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
|||||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||||
import {
|
import {
|
||||||
getInstalledEvenHubAppById,
|
getInstalledEvenHubAppById,
|
||||||
@@ -483,6 +483,7 @@ class DashboardController {
|
|||||||
this.applyVerticalPositionIfChanged();
|
this.applyVerticalPositionIfChanged();
|
||||||
this.applyAppLayoutsIfChanged();
|
this.applyAppLayoutsIfChanged();
|
||||||
this.syncAssistantBridgeIfChanged();
|
this.syncAssistantBridgeIfChanged();
|
||||||
|
this.retargetContinuousCapture();
|
||||||
this.syncLockScreenSettingIfChanged();
|
this.syncLockScreenSettingIfChanged();
|
||||||
// A Preview color or scaling change should show on the mirror at once,
|
// A Preview color or scaling change should show on the mirror at once,
|
||||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||||
@@ -2197,16 +2198,16 @@ class DashboardController {
|
|||||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||||
// A window that needs its own provider and language sets them for every
|
// A window that needs its own provider and language sets them for every
|
||||||
// capture while it is open, so push-to-talk still shares its stream.
|
// capture while it is open, so push-to-talk still shares its stream.
|
||||||
// Otherwise only Whisper server is told a language (its own setting).
|
// Otherwise use the shared transcription-language preference.
|
||||||
const choice = this.transcriptionChoice();
|
const choice = this.transcriptionChoice();
|
||||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||||
return {
|
return {
|
||||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||||
usePhoneMic: communicator === null,
|
usePhoneMic: communicator === null,
|
||||||
provider,
|
provider,
|
||||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||||
openAiApiKey: openAiApiKeySetting.get(),
|
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
||||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||||
whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerModel: whisperServerModelSetting.get(),
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
|||||||
import { onAnySettingChanged,
|
import { onAnySettingChanged,
|
||||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
||||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||||
@@ -416,6 +416,7 @@ export class IosPreviewController {
|
|||||||
const phone = readPhoneBatteryState()
|
const phone = readPhoneBatteryState()
|
||||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||||
this.offSettings = onAnySettingChanged(() => {
|
this.offSettings = onAnySettingChanged(() => {
|
||||||
|
this.syncContinuousCapture()
|
||||||
this.syncLockSetting()
|
this.syncLockSetting()
|
||||||
this.pushBrightness()
|
this.pushBrightness()
|
||||||
this.relayout()
|
this.relayout()
|
||||||
@@ -1198,14 +1199,13 @@ export class IosPreviewController {
|
|||||||
/**
|
/**
|
||||||
* The transcription provider setting and the keys the cloud ones need. A
|
* The transcription provider setting and the keys the cloud ones need. A
|
||||||
* window that needs its own provider and language (Translate) sets them for
|
* window that needs its own provider and language (Translate) sets them for
|
||||||
* every capture while it is open; otherwise only Whisper server is told a
|
* every capture while it is open; otherwise use the shared language preference.
|
||||||
* language (its own setting).
|
|
||||||
*/
|
*/
|
||||||
private voiceProvider(): VoiceProviderSelection {
|
private voiceProvider(): VoiceProviderSelection {
|
||||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-2
@@ -9,7 +9,7 @@ declare const java: any;
|
|||||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||||
* directory each model's `dirName` names.
|
* directory each model's `dirName` names.
|
||||||
*
|
*
|
||||||
* Two models, same shape:
|
* Downloadable models, same shape:
|
||||||
* - "moonshine": the original on-device option (three files). The model is
|
* - "moonshine": the original on-device option (three files). The model is
|
||||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||||
* location that earlier releases copied the bundled files to, so
|
* location that earlier releases copied the bundled files to, so
|
||||||
@@ -29,12 +29,17 @@ declare const java: any;
|
|||||||
* used for Moonshine below. Every hash here was computed locally from a
|
* used for Moonshine below. Every hash here was computed locally from a
|
||||||
* freshly downloaded copy of the file, not copied from a doc page.
|
* freshly downloaded copy of the file, not copied from a doc page.
|
||||||
*
|
*
|
||||||
|
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
|
||||||
|
* speech and auto-detection; English-only downloads remain untouched.
|
||||||
|
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
||||||
|
* vocabulary hash computed locally. URLs pin the source revision.
|
||||||
|
*
|
||||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||||
* here is multiple files rather than one; they download sequentially through
|
* here is multiple files rather than one; they download sequentially through
|
||||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
|
||||||
|
|
||||||
type AsrModelFile = {
|
type AsrModelFile = {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -105,6 +110,30 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
|||||||
],
|
],
|
||||||
totalBytes: 160626066,
|
totalBytes: 160626066,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
"whisper-base": {
|
||||||
|
label: "Whisper (multilingual, base)",
|
||||||
|
dirName: "sherpa-onnx-whisper-base-int8",
|
||||||
|
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
|
||||||
|
files: [
|
||||||
|
{
|
||||||
|
name: "base-encoder.int8.onnx",
|
||||||
|
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
|
||||||
|
sizeBytes: 29120534,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "base-decoder.int8.onnx",
|
||||||
|
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
|
||||||
|
sizeBytes: 130672026,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "base-tokens.txt",
|
||||||
|
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
||||||
|
sizeBytes: 816730,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
totalBytes: 160609290,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export type AsrModelState = {
|
export type AsrModelState = {
|
||||||
@@ -129,6 +158,7 @@ function freshRuntime(): ModelRuntime {
|
|||||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||||
moonshine: freshRuntime(),
|
moonshine: freshRuntime(),
|
||||||
"whisper-base-en": freshRuntime(),
|
"whisper-base-en": freshRuntime(),
|
||||||
|
"whisper-base": freshRuntime(),
|
||||||
};
|
};
|
||||||
|
|
||||||
function modelDirPath(id: AsrModelId): string {
|
function modelDirPath(id: AsrModelId): string {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
||||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||||
import { SonioxSttClient } from "./soniox-stt";
|
import { SonioxSttClient } from "./soniox-stt";
|
||||||
import type { VoiceProviderKind } from "./voice-control";
|
import type { VoiceProviderKind } from "./voice-control";
|
||||||
@@ -14,11 +14,12 @@ export type VoiceProviderSelection = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
/**
|
/**
|
||||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||||
* detect the language. Cloud providers only.
|
* detect the language. Also used by on-device Whisper.
|
||||||
*/
|
*/
|
||||||
language?: string;
|
language?: string;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
|
openAiTranscriptionModel?: OpenAiSttModel;
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
@@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
|||||||
whisper: {
|
whisper: {
|
||||||
name: "OpenAI",
|
name: "OpenAI",
|
||||||
apiKey: (selection) => selection.openAiApiKey,
|
apiKey: (selection) => selection.openAiApiKey,
|
||||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
||||||
},
|
},
|
||||||
"whisper-server": {
|
"whisper-server": {
|
||||||
name: "Whisper server",
|
name: "Whisper server",
|
||||||
@@ -76,8 +77,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
|||||||
* What a capture transcribes with: the provider and the language it is told
|
* What a capture transcribes with: the provider and the language it is told
|
||||||
* to expect. A running capture is shared only by holders asking for the same.
|
* to expect. A running capture is shared only by holders asking for the same.
|
||||||
*/
|
*/
|
||||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
||||||
|
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||||
|
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
* on the Realtime API, with a configurable transcription model).
|
||||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||||
*
|
*
|
||||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||||
@@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||||
const MODEL_ID = "gpt-realtime-whisper";
|
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
||||||
|
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
||||||
const TARGET_SAMPLE_RATE = 24000;
|
const TARGET_SAMPLE_RATE = 24000;
|
||||||
|
|
||||||
export type OpenAiSttOptions = CloudSttOptions;
|
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
||||||
|
|
||||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||||
private ws: SocketConnection | null = null;
|
private ws: SocketConnection | null = null;
|
||||||
@@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
onOpen: () => {
|
onOpen: () => {
|
||||||
if (this.closed) return;
|
if (this.closed) return;
|
||||||
this.open = true;
|
this.open = true;
|
||||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
||||||
for (const chunk of this.pendingChunks.splice(0)) {
|
for (const chunk of this.pendingChunks.splice(0)) {
|
||||||
this.sendChunk(chunk);
|
this.sendChunk(chunk);
|
||||||
}
|
}
|
||||||
@@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
this.options.onError(message.error?.code === "model_not_found"
|
||||||
|
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
||||||
|
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||||
return;
|
return;
|
||||||
default:
|
default:
|
||||||
return;
|
return;
|
||||||
@@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function sessionConfig(language: string): object {
|
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
||||||
return {
|
return {
|
||||||
type: "session.update",
|
type: "session.update",
|
||||||
session: {
|
session: {
|
||||||
@@ -209,7 +212,7 @@ function sessionConfig(language: string): object {
|
|||||||
audio: {
|
audio: {
|
||||||
input: {
|
input: {
|
||||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
transcription: { model, ...(language ? { language } : {}) },
|
||||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||||
turn_detection: null,
|
turn_detection: null,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
/** Languages supported by both Whisper backends, in picker order. */
|
||||||
|
export const TRANSCRIPTION_LANGUAGES = {
|
||||||
|
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
|
||||||
|
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
|
||||||
|
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
|
||||||
|
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
|
||||||
|
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
|
||||||
|
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
|
||||||
|
|
||||||
|
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
|
||||||
|
export function resolveTranscriptionLanguage(
|
||||||
|
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
||||||
|
): string {
|
||||||
|
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
||||||
|
if (provider === "onboard") return "";
|
||||||
|
if (choice === "default") {
|
||||||
|
if (provider === "onboard-whisper") return "en";
|
||||||
|
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
|
||||||
|
}
|
||||||
|
return choice === "auto" ? "" : choice;
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { OpenAiSttModel } from "./openai-stt";
|
||||||
import { Utils } from "@nativescript/core";
|
import { Utils } from "@nativescript/core";
|
||||||
|
|
||||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||||
@@ -24,7 +25,7 @@ export type VoiceControlState = {
|
|||||||
detail: string;
|
detail: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
||||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||||
@@ -54,11 +55,12 @@ export type PushToTalkOptions = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
|
openAiTranscriptionModel?: OpenAiSttModel;
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
whisperServerModel?: string;
|
whisperServerModel?: string;
|
||||||
whisperServerApiKey?: string;
|
whisperServerApiKey?: string;
|
||||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
|
||||||
language?: string;
|
language?: string;
|
||||||
saveRecording: boolean;
|
saveRecording: boolean;
|
||||||
/**
|
/**
|
||||||
@@ -396,6 +398,7 @@ export class FaceclawVoiceControlBridge {
|
|||||||
// Which on-device model to load; a no-op setter for every provider except
|
// Which on-device model to load; a no-op setter for every provider except
|
||||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||||
|
this.controller?.setOnboardLanguage(options.language ?? "en");
|
||||||
const captureId = this.nextNativeCaptureId++;
|
const captureId = this.nextNativeCaptureId++;
|
||||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||||
this.controller?.start("onboard", captureId);
|
this.controller?.start("onboard", captureId);
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { OpenAiSttModel } from "../native/openai-stt";
|
||||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||||
@@ -18,6 +19,7 @@ import {
|
|||||||
} from "~/assistant/models";
|
} from "~/assistant/models";
|
||||||
import { isLocalModelReady } from "../native/local-llm";
|
import { isLocalModelReady } from "../native/local-llm";
|
||||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||||
|
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
|
||||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||||
@@ -200,13 +202,6 @@ export class ConfigSettingEnum<TValue extends string, TId extends string = strin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const launcherLayoutSetting = new ConfigSettingEnum({
|
|
||||||
id: "launcher-layout", label: "Layout", storageKey: "launcher.layout",
|
|
||||||
defaultValue: "alphabetical", values: ["alphabetical", "custom"] as const,
|
|
||||||
formatValue: value => value === "custom" ? "Custom" : "Alphabetical",
|
|
||||||
description: "Sort apps and folders alphabetically, or arrange icons yourself. Custom positions are remembered when switching layouts.",
|
|
||||||
});
|
|
||||||
|
|
||||||
type ConfigSettingStringOptions<TId extends string> = ConfigSettingOptions<string, TId> & {
|
type ConfigSettingStringOptions<TId extends string> = ConfigSettingOptions<string, TId> & {
|
||||||
editorTitle?: string;
|
editorTitle?: string;
|
||||||
glassesEditTitle?: string;
|
glassesEditTitle?: string;
|
||||||
@@ -1143,9 +1138,35 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
|||||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
||||||
|
id: "openai-transcription-model",
|
||||||
|
label: "OpenAI transcription model",
|
||||||
|
storageKey: "voice.openAiTranscriptionModel",
|
||||||
|
defaultValue: "gpt-realtime-whisper",
|
||||||
|
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
||||||
|
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
||||||
|
});
|
||||||
|
|
||||||
|
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
||||||
|
id: "voice-language",
|
||||||
|
label: "Transcription language",
|
||||||
|
storageKey: "voice.language",
|
||||||
|
defaultValue: "default",
|
||||||
|
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
|
||||||
|
formatValue: (value) => value === "default" ? "Provider default"
|
||||||
|
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
||||||
|
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
||||||
|
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
||||||
|
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
|
||||||
|
});
|
||||||
|
|
||||||
|
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
||||||
|
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
|
||||||
|
}
|
||||||
|
|
||||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||||
id: "whisper-server-language",
|
id: "whisper-server-language",
|
||||||
label: "Whisper language",
|
label: "Whisper server language",
|
||||||
storageKey: "voice.whisperServerLanguage",
|
storageKey: "voice.whisperServerLanguage",
|
||||||
defaultValue: "",
|
defaultValue: "",
|
||||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||||
@@ -1153,7 +1174,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
|
|||||||
formatValue: (value) => value || "Auto-detect",
|
formatValue: (value) => value || "Auto-detect",
|
||||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||||
description:
|
description:
|
||||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ import {
|
|||||||
mapboxApiKeySetting,
|
mapboxApiKeySetting,
|
||||||
mirrorTouchSetting,
|
mirrorTouchSetting,
|
||||||
openAiApiKeySetting,
|
openAiApiKeySetting,
|
||||||
|
openAiTranscriptionModelSetting,
|
||||||
previewColorSetting,
|
previewColorSetting,
|
||||||
previewScalingSetting,
|
previewScalingSetting,
|
||||||
phoneRotationSetting,
|
phoneRotationSetting,
|
||||||
@@ -87,6 +88,7 @@ import {
|
|||||||
toggleSettingMenuItem,
|
toggleSettingMenuItem,
|
||||||
useMicControlSetting,
|
useMicControlSetting,
|
||||||
voiceProviderSetting,
|
voiceProviderSetting,
|
||||||
|
voiceLanguageSetting,
|
||||||
screenTimeoutSetting,
|
screenTimeoutSetting,
|
||||||
wakeToLastAppSetting,
|
wakeToLastAppSetting,
|
||||||
wakeWordActionSetting,
|
wakeWordActionSetting,
|
||||||
@@ -143,9 +145,11 @@ function settingsSections(): SettingsSection[] {
|
|||||||
items: [
|
items: [
|
||||||
enumSettingMenuItem(wakeWordActionSetting),
|
enumSettingMenuItem(wakeWordActionSetting),
|
||||||
enumSettingMenuItem(voiceProviderSetting),
|
enumSettingMenuItem(voiceProviderSetting),
|
||||||
|
enumSettingMenuItem(voiceLanguageSetting),
|
||||||
...whisperServerMenuItems(),
|
...whisperServerMenuItems(),
|
||||||
asrModelMenuItem("moonshine"),
|
asrModelMenuItem("moonshine"),
|
||||||
asrModelMenuItem("whisper-base-en"),
|
asrModelMenuItem("whisper-base-en"),
|
||||||
|
asrModelMenuItem("whisper-base"),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -284,7 +288,7 @@ function settingsSections(): SettingsSection[] {
|
|||||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||||
item.label !== screenTimeoutSetting.label) };
|
item.label !== screenTimeoutSetting.label) };
|
||||||
return section;
|
return section;
|
||||||
@@ -527,6 +531,7 @@ function asrModelStatusText(id: AsrModelId): string {
|
|||||||
*/
|
*/
|
||||||
function whisperServerMenuItems(): MenuItem[] {
|
function whisperServerMenuItems(): MenuItem[] {
|
||||||
return [
|
return [
|
||||||
|
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
||||||
textSettingMenuItem(whisperServerUrlSetting),
|
textSettingMenuItem(whisperServerUrlSetting),
|
||||||
textSettingMenuItem(whisperServerModelSetting),
|
textSettingMenuItem(whisperServerModelSetting),
|
||||||
textSettingMenuItem(whisperServerLanguageSetting),
|
textSettingMenuItem(whisperServerLanguageSetting),
|
||||||
|
|||||||
+2
-50
@@ -326,7 +326,7 @@ class ShellOverlayMenuLayer extends MenuLayer {
|
|||||||
*/
|
*/
|
||||||
keepWindowFocus = false;
|
keepWindowFocus = false;
|
||||||
|
|
||||||
constructor(items: MenuItem[], footer: string | undefined, private readonly onClosed: () => void, title = "System") {
|
constructor(items: MenuItem[], footer: string | undefined, private readonly onClosed: () => void) {
|
||||||
// Aligned to the min-height window band (like the sidebar), wherever the
|
// Aligned to the min-height window band (like the sidebar), wherever the
|
||||||
// vertical position setting currently puts it; centered over the
|
// vertical position setting currently puts it; centered over the
|
||||||
// application area, i.e. the part of the screen beside a side strip
|
// application area, i.e. the part of the screen beside a side strip
|
||||||
@@ -334,7 +334,7 @@ class ShellOverlayMenuLayer extends MenuLayer {
|
|||||||
// the strip overlays).
|
// the strip overlays).
|
||||||
const width = 272;
|
const width = 272;
|
||||||
const area = appViewportRect("min");
|
const area = appViewportRect("min");
|
||||||
super(title, items, {
|
super("System", items, {
|
||||||
x: area.x + (((area.width - width) / 2) | 0),
|
x: area.x + (((area.width - width) / 2) | 0),
|
||||||
y: area.y + 8,
|
y: area.y + 8,
|
||||||
width,
|
width,
|
||||||
@@ -1304,43 +1304,6 @@ class Shell {
|
|||||||
return hidden ? this.windows.filter((window) => window !== hidden) : this.windows;
|
return hidden ? this.windows.filter((window) => window !== hidden) : this.windows;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Reorder visible app windows while keeping pinned/temporary slots and foreground stable. */
|
|
||||||
private moveSwitcherWindow(windowId: string, targetId: string): boolean {
|
|
||||||
const movable = this.switcherWindows().filter(window => window.closeable);
|
|
||||||
const from = movable.findIndex(window => window.windowId === windowId);
|
|
||||||
const to = movable.findIndex(window => window.windowId === targetId);
|
|
||||||
if (from < 0 || to < 0 || from === to) return false;
|
|
||||||
const foreground = this.foregroundWindow();
|
|
||||||
const ids = new Set(movable.map(window => window.windowId));
|
|
||||||
const [moved] = movable.splice(from, 1);
|
|
||||||
movable.splice(to, 0, moved!);
|
|
||||||
let index = 0;
|
|
||||||
this.windows = this.windows.map(window => ids.has(window.windowId) ? movable[index++]! : window);
|
|
||||||
if (foreground) this.selectedIndex = this.windows.indexOf(foreground);
|
|
||||||
this.config.onWindowsChanged?.();
|
|
||||||
this.config.requestShellRender();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Capture window IDs: changing titles or closing windows cannot redirect a pending move. */
|
|
||||||
private openSwitcherMoveMenu(windowId: string): void {
|
|
||||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
|
||||||
const movable = this.switcherWindows().filter(window => window.closeable);
|
|
||||||
const selected = movable.findIndex(window => window.windowId === windowId);
|
|
||||||
if (selected < 0 || movable.length < 2) return;
|
|
||||||
const items: MenuItem[] = movable.map((window, index) => ({
|
|
||||||
label: `${index + 1}. ${window.title}`,
|
|
||||||
onSelect: ctx => {
|
|
||||||
ctx.stack.pop();
|
|
||||||
this.moveSwitcherWindow(windowId, window.windowId);
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
const layer = new ShellOverlayMenuLayer(items, undefined, () => this.yieldFocusToSidebar(), "Move app to position");
|
|
||||||
layer.selectItem(selected);
|
|
||||||
this.stack.push(layer);
|
|
||||||
this.config.requestShellRender();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The status bar items the switcher's selection can move onto, in order:
|
* The status bar items the switcher's selection can move onto, in order:
|
||||||
* with a bottom switcher (the selection reaches them past the last
|
* with a bottom switcher (the selection reaches them past the last
|
||||||
@@ -1996,17 +1959,6 @@ class Shell {
|
|||||||
},
|
},
|
||||||
}]),
|
}]),
|
||||||
);
|
);
|
||||||
const movableWindows = this.switcherWindows().filter(window => window.closeable);
|
|
||||||
if (movableWindows.length > 1 && movableWindows.includes(foreground)) {
|
|
||||||
items.push({
|
|
||||||
label: "Move in app switcher",
|
|
||||||
description: "Choose this open window's position in the app switcher.",
|
|
||||||
onSelect: ctx => {
|
|
||||||
ctx.stack.pop();
|
|
||||||
this.openSwitcherMoveMenu(foreground.windowId);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (brightnessSetting.get() !== "auto") {
|
if (brightnessSetting.get() !== "auto") {
|
||||||
items.push({
|
items.push({
|
||||||
label: "Brightness",
|
label: "Brightness",
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# Launcher folders and layouts
|
|
||||||
|
|
||||||
Open the Apps window's app menu (tap then hold by default). Layout and New folder are available even without a single icon selected. New folder creates a persistent empty folder with an unused generated name. Select a folder, or enter it, to rename it through the existing phone text editor. Rename commits on Submit; Cancel leaves the folder unchanged. Names must be non-empty, single-line, at most 24 characters, and unique ignoring case. Delete folder requires confirmation and returns its apps to the root without uninstalling them.
|
|
||||||
|
|
||||||
Layout offers Alphabetical (the existing default) and Custom. In Custom, select an app or folder icon and choose Move icon, navigate to its destination and click to place it at that position. Double tap cancels; the watch's back from the left edge also cancels. The phone mirror can select the destination too. Moves stay within the root or current folder; Move to folder and Remove from folder remain available for changing membership. Renaming a folder preserves its root position and internal order. New apps append alphabetically after saved icons. Switching to Alphabetical retains saved custom positions for later use.
|
|
||||||
|
|
||||||
Persistence uses the existing shared settings store. The legacy launcher.folders assignment map stays compatible; new launcher.folderNames metadata preserves explicitly created/renamed empty folders. Untouched legacy folders keep disappearing when their last app is removed. launcher.layout stores the mode, and launcher.order stores root and per-folder order. Assistant folder tools remain consistent with empty folders. Existing installations retain their grouping and alphabetical layout.
|
|
||||||
|
|
||||||
Software checks cover defaults, restart, existing assignments, empty folder creation/deletion, duplicate and invalid names, rename commit/cancel, independent root/folder positions, layout toggling, new/removed apps, malformed metadata, move cancellation and deletion confirmation. TypeScript and changed-source lint are checked separately.
|
|
||||||
|
|
||||||
Hardware validation remains pending: ring row/item selection, watch directional navigation and back, moving across scroll rows, phone-mirror placement, phone rename Submit/Cancel, reopening folders and app restart. No firmware was flashed.
|
|
||||||
|
|
||||||
Independent branch validation: 1073 JavaScript tests passed, 2 skipped, 0 failed; TypeScript and changed-source lint passed.
|
|
||||||
|
|
||||||
## Open app switcher order
|
|
||||||
|
|
||||||
The system menu adds Move in app switcher when the selected window can be moved and at least two app windows are open. Its numbered position picker shows the currently open app windows in switcher order, preselects the current position, and uses window IDs internally to distinguish windows with identical titles. Choose a destination to move there, or double tap to cancel.
|
|
||||||
|
|
||||||
This arranges existing open windows independently of the launcher's Alphabetical/Custom preference. Pinned windows keep their slots and temporary status-item windows do not participate. Reordering retains the same foreground window, attention state and most-recently-used history; closing a window still returns to the most recently viewed one. New windows append as before. Open-window order lasts for those windows' lifetime; reopening the app does not restore previously closed windows or write ephemeral window IDs to settings.
|
|
||||||
|
|
||||||
Software tests cover both directions, menu integration, duplicate titles, captured source IDs, cancellation, removed source/destination, temporary and pinned windows, appending/replacing windows, foreground identity and MRU close behavior. Hardware checks remain pending for ring/watch navigation in the position picker and left/right/bottom/popup switcher placements.
|
|
||||||
|
|
||||||
Independent branch after adding open-window ordering: 1080 JavaScript tests passed, 2 skipped; TypeScript and changed-source lint passed.
|
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Whisper transcription languages
|
||||||
|
|
||||||
|
Settings > Voice > Transcription language selects French, English, 27 other
|
||||||
|
languages, or auto-detection. It applies to ordinary dictation and continuous
|
||||||
|
transcription using OpenAI, on-device Whisper, and the other cloud providers.
|
||||||
|
Translate retains its own spoken-language preference. Moonshine remains
|
||||||
|
English-only; Apple's on-device speech recognizer retains its own locale.
|
||||||
|
|
||||||
|
The initial **Provider default** preserves existing installations: English for
|
||||||
|
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
|
||||||
|
Whisper server language for self-hosted Whisper. Selecting another shared
|
||||||
|
language overrides the server language without deleting it. Returning to Provider
|
||||||
|
default restores it. Settings persist through the existing settings store.
|
||||||
|
|
||||||
|
For French or auto-detection with **On-device (Whisper)**, download
|
||||||
|
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
|
||||||
|
English uses the existing English-only base model; neither model is deleted or
|
||||||
|
downloaded automatically. A missing model uses the existing “Voice model not
|
||||||
|
downloaded” status. Language changes restart active continuous transcription;
|
||||||
|
other captures use the new language when started.
|
||||||
|
|
||||||
|
## Software validation
|
||||||
|
|
||||||
|
- JavaScript regression tests exercise stored settings/reload, compatible
|
||||||
|
defaults, French and auto-detection, Translate overrides in Android/iOS capture
|
||||||
|
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
|
||||||
|
- Kotlin tests exercise English/French/other/automatic model selection and file
|
||||||
|
names. Android release compilation checks the actual sherpa-onnx bridge.
|
||||||
|
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
|
||||||
|
the existing resumable downloader. Encoder/decoder hashes and sizes come from
|
||||||
|
repository LFS metadata; the vocabulary hash was computed from its contents.
|
||||||
|
|
||||||
|
## Device validation still required
|
||||||
|
|
||||||
|
With both phone microphone preview and glasses microphone, check French dictation
|
||||||
|
(including accents), auto-detection, English rollback, continuous capture while
|
||||||
|
changing language, cancellation/resumption of model download, and persistence
|
||||||
|
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
|
||||||
|
No paid API call, microphone recording, or firmware update is needed for the
|
||||||
|
software checks above; recognition accuracy and latency require device testing.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||||
|
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
||||||
|
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
||||||
|
|
||||||
|
## OpenAI transcription model selection
|
||||||
|
|
||||||
|
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
||||||
|
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
||||||
|
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
||||||
|
The unset default retains gpt-realtime-whisper for existing installations; no
|
||||||
|
provider, language or Assistant model preference is overwritten. This setting
|
||||||
|
applies to Translate's speech recognition as well as Transcribe and dictation.
|
||||||
|
The two transcribe alternatives produce transcript deltas after a committed
|
||||||
|
audio segment; existing pause segmentation and push-to-talk release supply the
|
||||||
|
commits. Choosing a model retargets active continuous capture.
|
||||||
|
|
||||||
|
Model access errors name the requested model before the long project error and
|
||||||
|
point to the setting. Factory/session-wire tests cover model, language, manual
|
||||||
|
commit and final transcript behavior; settings tests cover persistence and the
|
||||||
|
Android/iOS option plumbing. No real API inference was performed; actual project
|
||||||
|
access, recognition and latency remain to be checked on a device. The model list
|
||||||
|
provided by the operator establishes account visibility, not a successful audio
|
||||||
|
request.
|
||||||
|
|
||||||
|
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
||||||
+18
@@ -0,0 +1,18 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
/** The English-only files remain usable after upgrading; other languages need multilingual base. */
|
||||||
|
data class WhisperModelFiles(val language: String) {
|
||||||
|
val englishOnly: Boolean get() = language == "en"
|
||||||
|
val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
|
||||||
|
private val prefix: String get() = if (englishOnly) "base.en" else "base"
|
||||||
|
val encoder: String get() = "$prefix-encoder.int8.onnx"
|
||||||
|
val decoder: String get() = "$prefix-decoder.int8.onnx"
|
||||||
|
val tokens: String get() = "$prefix-tokens.txt"
|
||||||
|
val files: List<String> get() = listOf(encoder, decoder, tokens)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
/** Null preserves the old English default; an empty code enables automatic detection. */
|
||||||
|
fun forLanguage(language: String?): WhisperModelFiles =
|
||||||
|
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
|
|||||||
ASR_MODELS: {
|
ASR_MODELS: {
|
||||||
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
||||||
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
||||||
|
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
|
||||||
},
|
},
|
||||||
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
||||||
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ function settingsFixture(store = new Map()) {
|
|||||||
};
|
};
|
||||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||||
|
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||||
}};
|
}};
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||||
|
|||||||
@@ -207,6 +207,8 @@ test('provider selection falls back to on-device without a key and connects the
|
|||||||
// A running capture is shared only on the same provider and language.
|
// A running capture is shared only on the same provider and language.
|
||||||
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
||||||
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
||||||
|
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
|
||||||
|
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
|
||||||
const statuses = [];
|
const statuses = [];
|
||||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||||
@@ -225,13 +227,13 @@ test('provider selection falls back to on-device without a key and connects the
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
||||||
for (const language of ['ja', '']) {
|
for (const language of ['fr', 'ja', '']) {
|
||||||
const elevenlabs = harness('elevenlabs', true, { language });
|
const elevenlabs = harness('elevenlabs', true, { language });
|
||||||
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
||||||
|
|
||||||
const soniox = harness('soniox', true, { language });
|
const soniox = harness('soniox', true, { language });
|
||||||
soniox.open();
|
soniox.open();
|
||||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
|
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
|
||||||
|
|
||||||
const openai = harness('openai', true, { language });
|
const openai = harness('openai', true, { language });
|
||||||
openai.open();
|
openai.open();
|
||||||
@@ -240,3 +242,43 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
|||||||
assert.equal(transcription.language, language || undefined);
|
assert.equal(transcription.language, language || undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
||||||
|
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
||||||
|
const h = harness('openai', true, { model, language: 'fr' });
|
||||||
|
h.open();
|
||||||
|
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
||||||
|
assert.equal(input.transcription.model, model);
|
||||||
|
assert.equal(input.transcription.language, 'fr');
|
||||||
|
assert.equal(input.turn_detection, null);
|
||||||
|
h.client.acceptPcm(pcm(1200, 150));
|
||||||
|
h.client.acceptPcm(pcm(0, 1500));
|
||||||
|
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
||||||
|
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
||||||
|
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
||||||
|
assert.equal(h.events.at(-1).isFinal, true);
|
||||||
|
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
||||||
|
assert.ok(h.errors[0].includes(model));
|
||||||
|
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
||||||
|
const created = [];
|
||||||
|
const load = loader({ setTimeout, clearTimeout }, {
|
||||||
|
'./openai-stt': {
|
||||||
|
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
||||||
|
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
||||||
|
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
||||||
|
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
||||||
|
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
||||||
|
assert.equal(created[0].language, 'fr');
|
||||||
|
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||||
|
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||||
|
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
||||||
|
});
|
||||||
@@ -12,6 +12,7 @@ function settingsFixture(store = new Map()) {
|
|||||||
};
|
};
|
||||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||||
|
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||||
}};
|
}};
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ const ts = require('typescript');
|
|||||||
const path = require('node:path');
|
const path = require('node:path');
|
||||||
// The provider settings ios-preview-controller hands the voice bridge.
|
// The provider settings ios-preview-controller hands the voice bridge.
|
||||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
getVoiceTranscriptionLanguage: () => '',
|
||||||
|
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||||
function load(file, context) {
|
function load(file, context) {
|
||||||
@@ -168,6 +169,7 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
|
|||||||
let cachedFont = 'Light';
|
let cachedFont = 'Light';
|
||||||
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
||||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||||
|
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
|
||||||
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
||||||
setTimeout: fn => tasks.push(fn),
|
setTimeout: fn => tasks.push(fn),
|
||||||
});
|
});
|
||||||
@@ -384,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
|||||||
recordTo: name => { bridge.recording = name; },
|
recordTo: name => { bridge.recording = name; },
|
||||||
setRawSession: session => { bridge.rawSession = session; } };
|
setRawSession: session => { bridge.rawSession = session; } };
|
||||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
|
||||||
saveTranscribeAudioSetting: { get: () => true } };
|
saveTranscribeAudioSetting: { get: () => true } };
|
||||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
import kotlin.test.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertFalse
|
||||||
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
|
class WhisperModelFilesTest {
|
||||||
|
@Test fun existingEnglishFilesRemainUsable() {
|
||||||
|
for (language in listOf(null, "en", " EN ")) {
|
||||||
|
val model = WhisperModelFiles.forLanguage(language)
|
||||||
|
assertTrue(model.englishOnly)
|
||||||
|
assertEquals("en", model.language)
|
||||||
|
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
|
||||||
|
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
|
||||||
|
for (language in listOf("fr", "de", "ja", " FR ")) {
|
||||||
|
val model = WhisperModelFiles.forLanguage(language)
|
||||||
|
assertFalse(model.englishOnly)
|
||||||
|
assertEquals(language.trim().lowercase(), model.language)
|
||||||
|
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
|
||||||
|
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
|
||||||
|
val model = WhisperModelFiles.forLanguage("")
|
||||||
|
assertFalse(model.englishOnly)
|
||||||
|
assertEquals("", model.language)
|
||||||
|
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,239 +0,0 @@
|
|||||||
const test = require('node:test');
|
|
||||||
const assert = require('node:assert/strict');
|
|
||||||
const fs = require('node:fs');
|
|
||||||
const vm = require('node:vm');
|
|
||||||
const ts = require('typescript');
|
|
||||||
|
|
||||||
function fixture(store = new Map()) {
|
|
||||||
const storage = { getStringSetting: (key, fallback) => store.get(key) ?? fallback,
|
|
||||||
setStringSetting: (key, value) => store.set(key, value), onSettingsStoreChanged() {},
|
|
||||||
getBooleanSetting: (_key, fallback) => fallback, setBooleanSetting() {} };
|
|
||||||
function load(file, deps, suffix = '') {
|
|
||||||
const context = { exports: {}, global: { isIOS: false }, console,
|
|
||||||
require: id => deps(id) };
|
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(file, 'utf8') + suffix, {
|
|
||||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
|
||||||
}).outputText, context);
|
|
||||||
return context.exports;
|
|
||||||
}
|
|
||||||
const layout = load('app/apps/launcher/launcher-layout.ts', () => storage);
|
|
||||||
const folders = load('app/apps/launcher/launcher-folders.ts', id => id === './launcher-layout' ? layout : storage);
|
|
||||||
const settings = load('app/ui/dashboard-settings.ts', id => {
|
|
||||||
if (id.includes('settings-store')) return storage;
|
|
||||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
|
||||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
|
||||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
|
||||||
});
|
|
||||||
return { store, layout, folders, settings, load };
|
|
||||||
}
|
|
||||||
const plain = value => JSON.parse(JSON.stringify(value));
|
|
||||||
|
|
||||||
test('legacy assignments and alphabetical default survive creation, rename and restart', () => {
|
|
||||||
const f = fixture();
|
|
||||||
const original = plain(f.folders.getFolderAssignments());
|
|
||||||
assert.equal(f.settings.launcherLayoutSetting.get(), 'alphabetical');
|
|
||||||
assert.equal(f.folders.createFolder(' Français '), true);
|
|
||||||
assert.deepEqual(plain(f.folders.getFolders().get('Français')), []);
|
|
||||||
assert.deepEqual(plain(f.folders.getFolderAssignments()), original);
|
|
||||||
assert.equal(f.folders.renameFolder('Games', 'Jeux'), true);
|
|
||||||
const again = fixture(f.store);
|
|
||||||
assert.equal(again.folders.getFolderAssignments().blocks, 'Jeux');
|
|
||||||
assert.equal(again.folders.getFolderAssignments().calculator, 'Prototypes');
|
|
||||||
assert.equal(again.folders.getFolders().has('Français'), true);
|
|
||||||
assert.equal(again.folders.getFolders().has('Games'), false);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('folder names reject blank, multiline, long and case-insensitive duplicate names', () => {
|
|
||||||
const f = fixture();
|
|
||||||
for (const name of ['', ' ', 'games', 'GAMES', 'x'.repeat(25), 'Two\nlines']) {
|
|
||||||
assert.equal(f.folders.createFolder(name), false, name);
|
|
||||||
assert.equal(f.folders.renameFolder('Prototypes', name), false, name);
|
|
||||||
}
|
|
||||||
assert.equal(f.folders.renameFolder('Games', 'games'), true);
|
|
||||||
assert.equal(f.folders.renameFolder('missing', 'New name'), false);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('explicit empty folders persist, can be deleted and never reintroduce default assignments', () => {
|
|
||||||
const f = fixture(new Map([['launcher.folders', '{}']]));
|
|
||||||
assert.equal(f.folders.createFolder('Empty'), true);
|
|
||||||
f.folders.setAppFolder('test', 'Empty');
|
|
||||||
f.folders.setAppFolder('test', null);
|
|
||||||
assert.equal(fixture(f.store).folders.getFolders().has('Empty'), true);
|
|
||||||
assert.equal(f.folders.disbandFolder('Empty'), 0);
|
|
||||||
assert.equal(f.folders.getFolders().size, 0);
|
|
||||||
assert.equal(f.folders.unusedNewFolderName(), 'New folder');
|
|
||||||
f.folders.createFolder('New folder');
|
|
||||||
assert.equal(f.folders.unusedNewFolderName(), 'New folder 2');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('deleting a non-empty folder ungroups all members without losing other assignments', () => {
|
|
||||||
const f = fixture();
|
|
||||||
assert.equal(f.folders.disbandFolder('Games'), 5);
|
|
||||||
assert.equal(f.folders.getFolderAssignments().blocks, undefined);
|
|
||||||
assert.equal(f.folders.getFolderAssignments().calculator, 'Prototypes');
|
|
||||||
assert.equal(fixture(f.store).folders.getFolders().has('Games'), false);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('custom root and folder orders persist independently, survive renames and append new apps', () => {
|
|
||||||
const f = fixture();
|
|
||||||
const entries = [{ label: 'Alpha', id: 'app:a' }, { label: 'Beta', id: 'app:b' }, { label: 'Games', id: 'folder:Games' }];
|
|
||||||
const sort = (scope, custom = true) => plain(f.layout.orderLauncherEntries(entries.slice(), e => e.id, scope, custom)).map(e => e.id);
|
|
||||||
f.settings.launcherLayoutSetting.set('custom');
|
|
||||||
f.layout.moveLauncherEntry(null, entries.map(e => e.id), 'folder:Games', 'app:a');
|
|
||||||
f.layout.setLauncherOrder('Games', ['app:b', 'app:a']);
|
|
||||||
assert.deepEqual(sort(null), ['folder:Games', 'app:a', 'app:b']);
|
|
||||||
assert.deepEqual(sort('Games'), ['app:b', 'app:a', 'folder:Games']);
|
|
||||||
assert.deepEqual(sort(null, false), ['app:a', 'app:b', 'folder:Games']);
|
|
||||||
f.settings.launcherLayoutSetting.set('alphabetical');
|
|
||||||
f.settings.launcherLayoutSetting.set('custom');
|
|
||||||
assert.deepEqual(sort(null), ['folder:Games', 'app:a', 'app:b']);
|
|
||||||
f.folders.renameFolder('Games', 'Jeux');
|
|
||||||
const again = fixture(f.store);
|
|
||||||
assert.equal(again.settings.launcherLayoutSetting.get(), 'custom');
|
|
||||||
entries[2] = { label: 'Jeux', id: 'folder:Jeux' };
|
|
||||||
assert.deepEqual(sort(null), ['folder:Jeux', 'app:a', 'app:b']);
|
|
||||||
assert.deepEqual(sort('Jeux'), ['app:b', 'app:a', 'folder:Jeux']);
|
|
||||||
entries.push({ label: 'A newly installed app', id: 'app:new' });
|
|
||||||
assert.deepEqual(sort(null), ['folder:Jeux', 'app:a', 'app:b', 'app:new']);
|
|
||||||
f.folders.disbandFolder('Jeux');
|
|
||||||
f.folders.createFolder('Jeux');
|
|
||||||
assert.deepEqual(sort('Jeux'), ['app:new', 'app:a', 'app:b', 'folder:Jeux']);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('malformed metadata is safe; moves tolerate removed apps and stale IDs', () => {
|
|
||||||
const f = fixture(new Map([['launcher.folderNames', '{bad'], ['launcher.order', '{bad']]));
|
|
||||||
assert.equal(f.folders.getExplicitFolderNames().length, 0);
|
|
||||||
assert.equal(f.layout.moveLauncherEntry(null, ['a', 'b'], 'missing', 'a'), false);
|
|
||||||
assert.equal(f.layout.moveLauncherEntry(null, ['a', 'b'], 'a', 'missing'), false);
|
|
||||||
assert.equal(f.layout.moveLauncherEntry(null, ['a', 'b'], 'a', 'b'), true);
|
|
||||||
assert.deepEqual(JSON.parse(f.store.get('launcher.order')).root, ['b', 'a']);
|
|
||||||
});
|
|
||||||
|
|
||||||
function launcherFixture() {
|
|
||||||
const f = fixture(new Map([['launcher.folders', '{}']]));
|
|
||||||
const layers = [], launches = [], deleted = [];
|
|
||||||
class IconGrid {
|
|
||||||
constructor(options) { this.options = options; this.index = null; }
|
|
||||||
get selectedItem() { return this.index === null ? null : this.options.items()[this.index] ?? null; }
|
|
||||||
resetSelection() { this.index = null; }
|
|
||||||
selectIndex(index) { this.index = index; }
|
|
||||||
enterRowMode() { this.index = null; }
|
|
||||||
async handleInput(event, ctx) {
|
|
||||||
if (event.type === 'click') await this.options.onActivate(this.selectedItem, ctx);
|
|
||||||
if (event.type === 'double-click' || event.type === 'swipe-left') this.options.onBack(ctx);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
class MenuLayer { constructor(title, items) { this.title = title; this.items = items; } }
|
|
||||||
class EditTextSettingLayer { constructor(setting) { this.setting = setting; } }
|
|
||||||
let edit;
|
|
||||||
const ctx = { stack: {
|
|
||||||
push: layer => layers.push(layer), pop() { const layer = layers.pop(); layer?.onRemoved?.(); },
|
|
||||||
popIfTop(test) { if (layers.length && test(layers.at(-1))) { this.pop(); return true; } return false; },
|
|
||||||
}, actions: { requestRender() {}, endTextSettingEdit() {}, startTextSettingsEdit(settings, title, submit, toggle, cancel) {
|
|
||||||
edit = { setting: settings[0], submit, cancel };
|
|
||||||
} } };
|
|
||||||
const api = f.load('app/apps/launcher/launcher-app.ts', id => {
|
|
||||||
if (id === './launcher-folders') return f.folders;
|
|
||||||
if (id === './launcher-layout') return f.layout;
|
|
||||||
if (id.endsWith('/dashboard-settings')) return { ...f.settings, EditTextSettingLayer,
|
|
||||||
enumSettingMenuItem: setting => ({ label: setting.label }) };
|
|
||||||
if (id.endsWith('/icon-grid')) return { IconGrid, ICON_GRID_BAND_INSET_X: 0, ICON_GRID_HIGHLIGHT_INSET_Y: 0 };
|
|
||||||
if (id.endsWith('/menu')) return { MenuLayer };
|
|
||||||
if (id.endsWith('/shell/shell')) return { shell: { yieldFocusToSidebar() {} } };
|
|
||||||
return {};
|
|
||||||
}, '\nexport { LauncherGridLayer };');
|
|
||||||
const apps = [{ appId: 'a', label: 'Alpha', icon: 'a' }, { appId: 'b', label: 'Beta', icon: 'b' }, { appId: 'c', label: 'Gamma', icon: 'c' }];
|
|
||||||
const layer = new api.LauncherGridLayer({ apps: () => apps, launchApp: id => launches.push(id), uninstallApp: id => deleted.push(id) });
|
|
||||||
return { ...f, layer, ctx, layers, launches, deleted, apps, edit: () => edit,
|
|
||||||
choose(label) { const item = layer.menuItems().find(item => item.label === label); assert.ok(item, label); item.onSelect(ctx); } };
|
|
||||||
}
|
|
||||||
|
|
||||||
test('launcher menu is available in row mode and supports empty folder creation and entry', async () => {
|
|
||||||
const h = launcherFixture();
|
|
||||||
assert.deepEqual(plain(h.layer.menuItems().map(item => item.label)), ['Layout', 'New folder']);
|
|
||||||
h.choose('New folder');
|
|
||||||
const folder = h.layer.entries().find(entry => entry.kind === 'folder');
|
|
||||||
assert.equal(folder.name, 'New folder');
|
|
||||||
await h.layer.openEntry(folder);
|
|
||||||
assert.equal(h.layer.currentFolder, 'New folder');
|
|
||||||
assert.equal(h.layer.entries().length, 0);
|
|
||||||
assert.ok(h.layer.menuItems().some(item => item.label === 'Rename folder'));
|
|
||||||
});
|
|
||||||
|
|
||||||
test('folder rename uses the existing editor, commits on submit only, and preserves open folder', async () => {
|
|
||||||
const h = launcherFixture(); h.choose('New folder');
|
|
||||||
await h.layer.openEntry(h.layer.entries().find(entry => entry.kind === 'folder'));
|
|
||||||
h.choose('Rename folder');
|
|
||||||
h.edit().setting.set('Renommé');
|
|
||||||
assert.equal(h.layer.menuItems().length, 0);
|
|
||||||
assert.equal(h.folders.getFolders().has('New folder'), true);
|
|
||||||
h.edit().cancel();
|
|
||||||
assert.equal(h.folders.getFolders().has('Renommé'), false);
|
|
||||||
h.choose('Rename folder');
|
|
||||||
h.edit().setting.set('Renommé');
|
|
||||||
h.edit().submit();
|
|
||||||
assert.equal(h.layer.currentFolder, 'Renommé');
|
|
||||||
assert.equal(h.folders.getFolders().has('New folder'), false);
|
|
||||||
assert.equal(h.layers.length, 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('moving icons reorders without launching; cancellation and alphabetical mode preserve order', async () => {
|
|
||||||
const h = launcherFixture();
|
|
||||||
h.layer.grid.selectIndex(0);
|
|
||||||
assert.equal(h.layer.menuItems().some(item => item.label === 'Move icon'), false);
|
|
||||||
h.settings.launcherLayoutSetting.set('custom');
|
|
||||||
h.choose('Move icon');
|
|
||||||
h.layer.grid.selectIndex(2);
|
|
||||||
await h.layer.handleInput({ type: 'click' }, h.ctx);
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'c', 'a']);
|
|
||||||
assert.equal(h.launches.length, 0);
|
|
||||||
h.choose('Move icon');
|
|
||||||
h.layer.grid.selectIndex(0);
|
|
||||||
await h.layer.handleInput({ type: 'double-click' }, h.ctx);
|
|
||||||
assert.equal(h.layer.moving, null);
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'c', 'a']);
|
|
||||||
h.settings.launcherLayoutSetting.set('alphabetical');
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['a', 'b', 'c']);
|
|
||||||
h.settings.launcherLayoutSetting.set('custom');
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'c', 'a']);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('folder moves stay inside their scope, folder deletion is confirmed and apps are not uninstalled', async () => {
|
|
||||||
const h = launcherFixture(); h.folders.createFolder('Dossier');
|
|
||||||
h.folders.setAppFolder('a', 'Dossier'); h.folders.setAppFolder('b', 'Dossier');
|
|
||||||
await h.layer.openEntry(h.layer.entries().find(entry => entry.kind === 'folder'));
|
|
||||||
h.settings.launcherLayoutSetting.set('custom'); h.layer.grid.selectIndex(0);
|
|
||||||
h.choose('Move icon'); await h.layer.openEntry(h.layer.entries()[1]);
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'a']);
|
|
||||||
h.choose('Delete folder');
|
|
||||||
assert.equal(h.folders.getFolders().has('Dossier'), true);
|
|
||||||
h.layers.at(-1).items[0].onSelect(h.ctx);
|
|
||||||
assert.equal(h.folders.getFolders().has('Dossier'), false);
|
|
||||||
assert.equal(h.layer.currentFolder, null);
|
|
||||||
assert.equal(h.layer.entries().length, 3);
|
|
||||||
assert.equal(h.deleted.length, 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
test('legacy folders with removed apps disappear while explicit empty folders remain', async () => {
|
|
||||||
const h = launcherFixture();
|
|
||||||
h.folders.setAppFolder('a', 'Legacy');
|
|
||||||
await h.layer.openEntry(h.layer.entries().find(entry => entry.kind === 'folder'));
|
|
||||||
h.apps.splice(0, 1);
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'c']);
|
|
||||||
assert.equal(h.layer.currentFolder, null);
|
|
||||||
h.folders.createFolder('Empty');
|
|
||||||
await h.layer.openEntry(h.layer.entries().find(entry => entry.kind === 'folder'));
|
|
||||||
assert.equal(h.layer.entries().length, 0);
|
|
||||||
assert.equal(h.layer.currentFolder, 'Empty');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('removing the source during a move cancels without launching or corrupting order', async () => {
|
|
||||||
const h = launcherFixture(); h.settings.launcherLayoutSetting.set('custom');
|
|
||||||
h.layer.grid.selectIndex(0); h.choose('Move icon');
|
|
||||||
h.apps.splice(0, 1);
|
|
||||||
await h.layer.openEntry(h.layer.entries()[0]);
|
|
||||||
assert.equal(h.layer.moving, null);
|
|
||||||
assert.equal(h.launches.length, 0);
|
|
||||||
assert.deepEqual(plain(h.layer.entries().map(entry => entry.appId)), ['b', 'c']);
|
|
||||||
});
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
const test = require('node:test');
|
|
||||||
const assert = require('node:assert/strict');
|
|
||||||
const fs = require('node:fs');
|
|
||||||
const vm = require('node:vm');
|
|
||||||
const ts = require('typescript');
|
|
||||||
|
|
||||||
function fixture() {
|
|
||||||
const source = ts.createSourceFile('shell.ts', fs.readFileSync('app/ui/shell/shell.ts', 'utf8'), ts.ScriptTarget.Latest, true);
|
|
||||||
const cls = source.statements.find(n => ts.isClassDeclaration(n) && n.name.text === 'Shell');
|
|
||||||
const names = new Set(['moveSwitcherWindow', 'openSwitcherMoveMenu', 'switcherWindows', 'foregroundWindow',
|
|
||||||
'openEscapeMenu', 'registerWindow', 'removeWindow', 'noteWindowVisible', 'mostRecentWindowIndex']);
|
|
||||||
const methods = cls.members.filter(n => names.has(n.name?.getText(source)));
|
|
||||||
assert.equal(methods.length, names.size);
|
|
||||||
const overlay = source.statements.find(n => ts.isClassDeclaration(n) && n.name.text === 'ShellOverlayMenuLayer');
|
|
||||||
class MenuLayer {
|
|
||||||
constructor(title, items) { this.title = title; this.items = items; }
|
|
||||||
selectItem(index) { this.selectedIndex = index; }
|
|
||||||
}
|
|
||||||
const context = { exports: {}, MenuLayer, appViewportRect: () => ({ x: 0, y: 0, width: 576 }),
|
|
||||||
CONTEXT_MENU_DIM: 0.5, brightnessSetting: { get: () => 'auto' }, makeInputEvent: e => e,
|
|
||||||
appFullscreenSetting: () => ({ get: () => false }), appKeepAwakeSetting: () => ({ label: 'Keep screen awake' }),
|
|
||||||
toggleSettingMenuItem: setting => ({ label: setting.label }),
|
|
||||||
swapMenuGesturesSetting: { get: () => false }, GESTURE_SHORT_THEN_LONG_PRESS: 'tap-hold',
|
|
||||||
};
|
|
||||||
vm.runInNewContext(ts.transpileModule(`${overlay.getText(source)}\nexport class Shell { ${methods.map(n => n.getText(source)).join('\n')} }`, {
|
|
||||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
|
||||||
}).outputText, context);
|
|
||||||
const s = new context.exports.Shell();
|
|
||||||
const foregroundCalls = [], closed = [], input = [];
|
|
||||||
const window = (windowId, appId = windowId, closeable = true, title = windowId) => ({
|
|
||||||
windowId, appId, closeable, title, requestRender() {},
|
|
||||||
setForeground: value => foregroundCalls.push([windowId, value]),
|
|
||||||
close: () => closed.push(windowId), handleInput: event => input.push([windowId, event.type]),
|
|
||||||
});
|
|
||||||
const launcher = window('launcher', 'launcher', false, 'Apps');
|
|
||||||
const first = window('terminal:one', 'terminal', true, 'Terminal');
|
|
||||||
const second = window('terminal:two', 'terminal', true, 'Terminal');
|
|
||||||
const third = window('calendar', 'calendar', true, 'Calendar');
|
|
||||||
const layers = [];
|
|
||||||
let changed = 0, renders = 0;
|
|
||||||
Object.assign(s, {
|
|
||||||
windows: [launcher, first, second, third], selectedIndex: 1, focus: 'sidebar', screenOn: true,
|
|
||||||
attention: new Map(), mruWindowIds: ['terminal:one', 'calendar', 'terminal:two', 'launcher'],
|
|
||||||
config: { voiceInputEnabled: false, onWindowsChanged: () => changed++, requestShellRender: () => renders++ },
|
|
||||||
yieldFocusToSidebar() { this.focus = 'sidebar'; },
|
|
||||||
stack: { isAtBase: () => !layers.length, push: layer => layers.push(layer),
|
|
||||||
pop: () => layers.pop()?.onRemoved?.() },
|
|
||||||
});
|
|
||||||
return { s, launcher, first, second, third, window, foregroundCalls, closed, input,
|
|
||||||
ids: () => Array.from(s.windows, w => w.windowId), changed: () => changed, renders: () => renders,
|
|
||||||
menu: () => layers.at(-1), choose(label) {
|
|
||||||
const item = layers.at(-1).items.find(item => item.label === label);
|
|
||||||
assert.ok(item, label); item.onSelect({ stack: s.stack });
|
|
||||||
} };
|
|
||||||
}
|
|
||||||
|
|
||||||
test('moving open windows in either direction keeps foreground, MRU and pinned launcher stable', () => {
|
|
||||||
const h = fixture(); h.s.focus = 'window';
|
|
||||||
const mru = Array.from(h.s.mruWindowIds);
|
|
||||||
assert.equal(h.s.moveSwitcherWindow(h.first.windowId, h.third.windowId), true);
|
|
||||||
assert.deepEqual(h.ids(), ['launcher', 'terminal:two', 'calendar', 'terminal:one']);
|
|
||||||
assert.equal(h.s.foregroundWindow(), h.first);
|
|
||||||
assert.equal(h.s.selectedIndex, 3);
|
|
||||||
assert.equal(h.s.focus, 'window');
|
|
||||||
assert.deepEqual(Array.from(h.s.mruWindowIds), mru);
|
|
||||||
assert.deepEqual(h.foregroundCalls, []);
|
|
||||||
assert.deepEqual(h.closed, []);
|
|
||||||
assert.equal(h.changed(), 1); assert.equal(h.renders(), 1);
|
|
||||||
assert.equal(h.s.moveSwitcherWindow(h.first.windowId, h.second.windowId), true);
|
|
||||||
assert.deepEqual(h.ids(), ['launcher', 'terminal:one', 'terminal:two', 'calendar']);
|
|
||||||
assert.equal(h.s.foregroundWindow(), h.first);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('system menu opens a numbered position picker and moves the captured window, even with duplicate titles', () => {
|
|
||||||
const h = fixture();
|
|
||||||
h.s.openEscapeMenu(); h.choose('Move in app switcher');
|
|
||||||
assert.equal(h.menu().title, 'Move app to position');
|
|
||||||
assert.deepEqual(Array.from(h.menu().items, item => item.label), ['1. Terminal', '2. Terminal', '3. Calendar']);
|
|
||||||
assert.equal(h.menu().selectedIndex, 0);
|
|
||||||
// A delayed focus change must not move a different app than the menu captured.
|
|
||||||
h.s.selectedIndex = 2;
|
|
||||||
h.choose('3. Calendar');
|
|
||||||
assert.deepEqual(h.ids(), ['launcher', 'terminal:two', 'calendar', 'terminal:one']);
|
|
||||||
assert.equal(h.s.foregroundWindow(), h.second);
|
|
||||||
assert.equal(h.menu(), undefined);
|
|
||||||
assert.equal(h.s.focus, 'sidebar');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('cancelling the picker and choosing the current slot leave ordering unchanged', () => {
|
|
||||||
const h = fixture(); const before = h.ids();
|
|
||||||
h.s.openSwitcherMoveMenu(h.first.windowId); h.s.stack.pop();
|
|
||||||
assert.deepEqual(h.ids(), before);
|
|
||||||
h.s.openSwitcherMoveMenu(h.first.windowId); h.choose('1. Terminal');
|
|
||||||
assert.deepEqual(h.ids(), before);
|
|
||||||
assert.equal(h.changed(), 0);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('closing the pending source or destination prevents a stale move from affecting another window', () => {
|
|
||||||
for (const removed of ['terminal:one', 'calendar']) {
|
|
||||||
const h = fixture(); h.s.openSwitcherMoveMenu(h.first.windowId);
|
|
||||||
h.s.removeWindow(removed);
|
|
||||||
const afterClose = h.ids();
|
|
||||||
h.choose('3. Calendar');
|
|
||||||
assert.deepEqual(h.ids(), afterClose);
|
|
||||||
assert.equal(h.menu(), undefined);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test('temporary status windows and pinned slots stay fixed, including selected status foreground', () => {
|
|
||||||
const h = fixture();
|
|
||||||
const hidden = h.window('notifications:temporary');
|
|
||||||
const pinned = h.window('another-pinned', 'pinned', false);
|
|
||||||
h.s.windows.splice(2, 0, hidden, pinned);
|
|
||||||
h.s.selectedIndex = 2;
|
|
||||||
h.s.selectedStatus = { transient: true, host: hidden, item: { key: 'notification' } };
|
|
||||||
assert.equal(h.s.moveSwitcherWindow(hidden.windowId, h.first.windowId), false);
|
|
||||||
h.s.moveSwitcherWindow(h.first.windowId, h.third.windowId);
|
|
||||||
assert.deepEqual(h.ids(), ['launcher', 'terminal:two', 'notifications:temporary', 'another-pinned', 'calendar', 'terminal:one']);
|
|
||||||
assert.equal(h.s.foregroundWindow(), hidden);
|
|
||||||
assert.equal(h.s.selectedStatus.host, hidden);
|
|
||||||
h.s.openEscapeMenu();
|
|
||||||
assert.ok(!h.menu().items.some(item => item.label === 'Move in app switcher'));
|
|
||||||
h.s.stack.pop();
|
|
||||||
h.s.openSwitcherMoveMenu(h.first.windowId);
|
|
||||||
assert.deepEqual(Array.from(h.menu().items, item => item.label), ['1. Terminal', '2. Calendar', '3. Terminal']);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('new windows append, replacements keep their slot, and closing uses unchanged recency after a move', () => {
|
|
||||||
const h = fixture(); h.s.moveSwitcherWindow(h.first.windowId, h.third.windowId);
|
|
||||||
h.s.registerWindow(h.window('new'));
|
|
||||||
assert.equal(h.ids().at(-1), 'new');
|
|
||||||
const replacement = h.window(h.second.windowId, 'terminal', true, 'Renamed session');
|
|
||||||
h.s.registerWindow(replacement);
|
|
||||||
assert.equal(h.s.windows[1], replacement);
|
|
||||||
assert.equal(h.s.foregroundWindow(), h.first);
|
|
||||||
h.s.removeWindow(h.first.windowId);
|
|
||||||
assert.equal(h.s.foregroundWindow(), h.third);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('pinned, missing, single-window, screen-off and overlay cases cannot start a reorder', () => {
|
|
||||||
const h = fixture(); const before = h.ids();
|
|
||||||
for (const [from, to] of [['launcher', 'calendar'], ['calendar', 'launcher'], ['missing', 'calendar']]) {
|
|
||||||
assert.equal(h.s.moveSwitcherWindow(from, to), false);
|
|
||||||
}
|
|
||||||
h.s.openSwitcherMoveMenu('missing'); assert.equal(h.menu(), undefined);
|
|
||||||
h.s.screenOn = false; h.s.openSwitcherMoveMenu(h.first.windowId); assert.equal(h.menu(), undefined);
|
|
||||||
h.s.screenOn = true; h.s.stack.push({});
|
|
||||||
h.s.openSwitcherMoveMenu(h.first.windowId); assert.equal(h.menu().title, undefined); h.s.stack.pop();
|
|
||||||
assert.deepEqual(h.ids(), before);
|
|
||||||
h.s.selectedIndex = 0; h.s.openEscapeMenu();
|
|
||||||
assert.ok(!h.menu().items.some(item => item.label === 'Move in app switcher')); h.s.stack.pop();
|
|
||||||
h.s.windows = [h.launcher, h.first]; h.s.selectedIndex = 1; h.s.openEscapeMenu();
|
|
||||||
assert.ok(!h.menu().items.some(item => item.label === 'Move in app switcher'));
|
|
||||||
});
|
|
||||||
@@ -28,6 +28,7 @@
|
|||||||
"../app/ui/menu-core.ts",
|
"../app/ui/menu-core.ts",
|
||||||
"../app/phone-ui/phone-gestures.ts",
|
"../app/phone-ui/phone-gestures.ts",
|
||||||
"../app/native/java-direct-buffer.ts",
|
"../app/native/java-direct-buffer.ts",
|
||||||
|
"../app/native/transcription-language.ts",
|
||||||
"../app/native/notification-types.ts",
|
"../app/native/notification-types.ts",
|
||||||
"../app/g2/firmware-compat.ts",
|
"../app/g2/firmware-compat.ts",
|
||||||
"../app/g2/glasses-hardware-identity.ts",
|
"../app/g2/glasses-hardware-identity.ts",
|
||||||
|
|||||||
@@ -163,7 +163,8 @@ function androidBridge() {
|
|||||||
starts: 0,
|
starts: 0,
|
||||||
setListener(listener) { this.listener = listener; },
|
setListener(listener) { this.listener = listener; },
|
||||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
|
||||||
|
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
|
||||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||||
stop() { this.active = false; },
|
stop() { this.active = false; },
|
||||||
isCapturing() { return this.active; },
|
isCapturing() { return this.active; },
|
||||||
@@ -171,7 +172,8 @@ function androidBridge() {
|
|||||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
'./cloud-stt-provider': { createCloudSttClient: () => null,
|
||||||
|
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
|
||||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||||
FaceclawVoiceController: function () { return controller; },
|
FaceclawVoiceController: function () { return controller; },
|
||||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||||
@@ -179,6 +181,26 @@ function androidBridge() {
|
|||||||
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
|
||||||
|
const { bridge, controller } = androidBridge();
|
||||||
|
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
|
||||||
|
assert.equal(controller.modelKind, 'whisper');
|
||||||
|
assert.equal(controller.language, 'fr');
|
||||||
|
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
|
||||||
|
assert.equal(controller.starts, 1);
|
||||||
|
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
|
||||||
|
assert.equal(controller.language, '');
|
||||||
|
assert.equal(controller.starts, 2);
|
||||||
|
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
||||||
|
// An omitted language preserves the native bridge's original English behavior.
|
||||||
|
assert.equal(controller.language, 'en');
|
||||||
|
assert.equal(controller.starts, 3);
|
||||||
|
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
|
||||||
|
assert.equal(controller.language, 'en');
|
||||||
|
assert.equal(controller.starts, 3);
|
||||||
|
bridge.stop();
|
||||||
|
});
|
||||||
|
|
||||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||||
const { bridge, controller } = androidBridge();
|
const { bridge, controller } = androidBridge();
|
||||||
const options = { provider: 'onboard-whisper' };
|
const options = { provider: 'onboard-whisper' };
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const vm = require('node:vm');
|
||||||
|
const ts = require('typescript');
|
||||||
|
|
||||||
|
function load(file, requireModule = () => ({}), globals = {}) {
|
||||||
|
const exports = {};
|
||||||
|
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||||
|
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||||
|
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
|
||||||
|
return exports;
|
||||||
|
}
|
||||||
|
const languages = load('app/native/transcription-language.ts');
|
||||||
|
function settings(values = new Map()) {
|
||||||
|
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
|
||||||
|
: name.includes('settings-store') ? {
|
||||||
|
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
||||||
|
setStringSetting: (key, value) => values.set(key, value),
|
||||||
|
getBooleanSetting: (_key, fallback) => fallback,
|
||||||
|
onSettingsStoreChanged() {},
|
||||||
|
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
|
||||||
|
}
|
||||||
|
|
||||||
|
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
|
||||||
|
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||||
|
const s = settings(values);
|
||||||
|
assert.equal(s.voiceLanguageSetting.get(), 'default');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||||
|
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
|
||||||
|
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||||
|
let s = settings(values);
|
||||||
|
s.voiceLanguageSetting.set('fr');
|
||||||
|
s = settings(values);
|
||||||
|
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
|
||||||
|
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
|
||||||
|
}
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
|
||||||
|
s.voiceLanguageSetting.set('auto');
|
||||||
|
s = settings(values);
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||||
|
s.voiceLanguageSetting.set('default');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('corrupt language preferences revert to the compatible provider defaults', () => {
|
||||||
|
const s = settings(new Map([['voice.language', 'invalid']]));
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||||
|
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
|
||||||
|
const s = settings();
|
||||||
|
s.voiceLanguageSetting.set('fr');
|
||||||
|
s.voiceProviderSetting.set('whisper');
|
||||||
|
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||||
|
for (const [file, name, method] of [
|
||||||
|
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
||||||
|
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
||||||
|
]) {
|
||||||
|
const host = { continuousClients: new Map() };
|
||||||
|
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
|
||||||
|
// Run each real capture-options method with only its native dependencies stubbed.
|
||||||
|
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||||
|
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||||
|
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
|
||||||
|
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
||||||
|
const capture = loadBody(member.body.getText(tree), s, host);
|
||||||
|
assert.equal(capture().language, 'fr');
|
||||||
|
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||||
|
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
||||||
|
assert.equal(capture().provider, 'soniox');
|
||||||
|
assert.equal(capture().language, 'ja');
|
||||||
|
host.continuousClients.clear();
|
||||||
|
assert.equal(capture().language, 'fr');
|
||||||
|
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
function loadBody(body, s, host) {
|
||||||
|
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
|
||||||
|
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
test('multilingual downloads and deletion leave an existing English model intact', () => {
|
||||||
|
const files = new Map(), downloads = [];
|
||||||
|
const asr = load('app/native/asr-model.ts', () => ({
|
||||||
|
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
||||||
|
}), {
|
||||||
|
java: { io: { File: class {
|
||||||
|
constructor(name) { this.name = name; }
|
||||||
|
exists() { return files.has(this.name); }
|
||||||
|
length() { return files.get(this.name) ?? 0; }
|
||||||
|
delete() { return files.delete(this.name); }
|
||||||
|
} } },
|
||||||
|
com: { faceclaw: { app: {
|
||||||
|
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
||||||
|
FaceclawModelDownloader: class {
|
||||||
|
constructor(url, destination, hash, size, listener) {
|
||||||
|
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
||||||
|
}
|
||||||
|
start() {}
|
||||||
|
cancel() {}
|
||||||
|
},
|
||||||
|
} } },
|
||||||
|
});
|
||||||
|
const english = asr.ASR_MODELS['whisper-base-en'];
|
||||||
|
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
|
||||||
|
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||||
|
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||||
|
asr.startAsrModelDownload('whisper-base');
|
||||||
|
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
|
||||||
|
for (let index = 0; index < 3; index++) {
|
||||||
|
const download = downloads[index];
|
||||||
|
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
|
||||||
|
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
||||||
|
files.set(download.destination, download.size);
|
||||||
|
download.listener.onDone();
|
||||||
|
}
|
||||||
|
assert.equal(downloads.length, 3);
|
||||||
|
assert.equal(asr.isAsrModelReady('whisper-base'), true);
|
||||||
|
asr.deleteAsrModel('whisper-base');
|
||||||
|
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||||
|
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
||||||
|
const values = new Map();
|
||||||
|
let s = settings(values);
|
||||||
|
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||||
|
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||||
|
s.voiceLanguageSetting.set('fr');
|
||||||
|
s = settings(values);
|
||||||
|
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
||||||
|
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
||||||
|
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||||
|
});
|
||||||
Reference in new issue
Block a user