Compare commits

..
34 changed files with 400 additions and 512 deletions

No files matched your search

@@ -25,21 +25,18 @@ internal object AndroidSpeechEngines {
private const val FEATURE_DIM = 80 private const val FEATURE_DIM = 80
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */ /** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
fun recognizerConfig( fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
modelDir: File, kind: VoiceModelKind,
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
): OfflineRecognizerConfig {
val modelConfig = OfflineModelConfig.builder() val modelConfig = OfflineModelConfig.builder()
.setNumThreads(1) .setNumThreads(1)
if (kind == VoiceModelKind.WHISPER) { if (kind == VoiceModelKind.WHISPER) {
modelConfig modelConfig
.setWhisper(OfflineWhisperModelConfig.builder() .setWhisper(OfflineWhisperModelConfig.builder()
.setEncoder(File(modelDir, whisper.encoder).absolutePath) .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
.setDecoder(File(modelDir, whisper.decoder).absolutePath) .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
.setLanguage(whisper.language) .setLanguage("en")
.setTask("transcribe") .setTask("transcribe")
.build()) .build())
.setTokens(File(modelDir, whisper.tokens).absolutePath) .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
} else { } else {
modelConfig modelConfig
.setMoonshine(OfflineMoonshineModelConfig.builder() .setMoonshine(OfflineMoonshineModelConfig.builder()
@@ -504,6 +504,18 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
} }
} }
@JvmStatic
fun dismissAllNotifications(): Boolean {
val service = activeService ?: return false
return try {
service.cancelAllNotifications()
true
} catch (t: Throwable) {
Log.w(TAG, "failed to dismiss all notifications", t)
false
}
}
private fun emitNotificationPosted(key: String?) { private fun emitNotificationPosted(key: String?) {
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) } emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
} }
@@ -32,6 +32,17 @@ class FaceclawVoiceController(context: Context) {
"decoder_model_merged.ort", "decoder_model_merged.ort",
"tokens.txt" "tokens.txt"
) )
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
// Whisper re-encodes the whole buffer on every call, so the shared session
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
private val ASR_WHISPER_MODEL_FILES = arrayOf(
"base.en-encoder.int8.onnx",
"base.en-decoder.int8.onnx",
"base.en-tokens.txt"
)
// 50 ms chunks match the G2 packet cadence the rest of the pipeline // 50 ms chunks match the G2 packet cadence the rest of the pipeline
// (endpointing, transcript pacing) is tuned for. // (endpointing, transcript pacing) is tuned for.
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20 private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
@@ -73,7 +84,6 @@ class FaceclawVoiceController(context: Context) {
private val mainHandler = Handler(Looper.getMainLooper()) private val mainHandler = Handler(Looper.getMainLooper())
@Volatile @Volatile
private var communicator: FaceclawBleCommunicator? = null private var communicator: FaceclawBleCommunicator? = null
private var whisperModel = WhisperModelFiles.forLanguage(null)
private val host = object : VoiceCaptureHost { private val host = object : VoiceCaptureHost {
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
@@ -112,7 +122,7 @@ class FaceclawVoiceController(context: Context) {
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber { override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing") val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel)) return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
} }
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder { override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
@@ -158,11 +168,6 @@ class FaceclawVoiceController(context: Context) {
session.setOnboardModelKind(kind) session.setOnboardModelKind(kind)
} }
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
fun setOnboardLanguage(language: String?) {
whisperModel = WhisperModelFiles.forLanguage(language)
}
/** /**
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
* speaker stops. Used by hands-free ("Hey Even") capture, which has no * speaker stops. Used by hands-free ("Hey Even") capture, which has no
@@ -258,8 +263,8 @@ class FaceclawVoiceController(context: Context) {
*/ */
private fun findAsrModelDir(kind: VoiceModelKind): File? { private fun findAsrModelDir(kind: VoiceModelKind): File? {
val whisper = kind == VoiceModelKind.WHISPER val whisper = kind == VoiceModelKind.WHISPER
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList() val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName) val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
for (fileName in fileNames) { for (fileName in fileNames) {
val file = File(modelDir, fileName) val file = File(modelDir, fileName)
+10 -1
View File
@@ -1,6 +1,6 @@
import { NotificationsListLayer } from "../../ui/notifications"; import { NotificationsListLayer } from "../../ui/notifications";
import { NotificationFilterLayer } from "../../ui/notification-filter"; import { NotificationFilterLayer } from "../../ui/notification-filter";
import { onAndroidNotificationsChanged } from "../../native/notification-icons"; import { ALL_NOTIFICATIONS, dismissAllNotifications, onAndroidNotificationsChanged, readActiveNotifications } from "../../native/notification-icons";
import { import {
createInProcessWindow, createInProcessWindow,
YieldAtRootLayer, YieldAtRootLayer,
@@ -32,6 +32,15 @@ export function createNotificationsAppWindow(options: InProcessAppOptions): InPr
ctx.stack.pop(); ctx.stack.pop();
ctx.stack.push(new NotificationFilterLayer()); ctx.stack.push(new NotificationFilterLayer());
}, },
}, {
label: "Dismiss all notifications",
description: "Dismiss notifications on the phone and glasses. System-protected notifications may remain.",
disabled: () => readActiveNotifications(ALL_NOTIFICATIONS).length === 0,
onSelect: (ctx) => {
ctx.stack.pop();
dismissAllNotifications();
created.requestRender();
},
}], }],
actions: options.actions, actions: options.actions,
baseLayer: new YieldAtRootLayer(new NotificationsListLayer()), baseLayer: new YieldAtRootLayer(new NotificationsListLayer()),
+4 -5
View File
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import { import {
getInstalledEvenHubAppById, getInstalledEvenHubAppById,
@@ -483,7 +483,6 @@ class DashboardController {
this.applyVerticalPositionIfChanged(); this.applyVerticalPositionIfChanged();
this.applyAppLayoutsIfChanged(); this.applyAppLayoutsIfChanged();
this.syncAssistantBridgeIfChanged(); this.syncAssistantBridgeIfChanged();
this.retargetContinuousCapture();
this.syncLockScreenSettingIfChanged(); this.syncLockScreenSettingIfChanged();
// A Preview color or scaling change should show on the mirror at once, // A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on // not at the next composited frame (cheap: floor-limited, dedup'd on
@@ -2198,16 +2197,16 @@ class DashboardController {
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
// A window that needs its own provider and language sets them for every // A window that needs its own provider and language sets them for every
// capture while it is open, so push-to-talk still shares its stream. // capture while it is open, so push-to-talk still shares its stream.
// Otherwise use the shared transcription-language preference. // Otherwise only Whisper server is told a language (its own setting).
const choice = this.transcriptionChoice(); const choice = this.transcriptionChoice();
const provider = choice?.provider ?? voiceProviderSetting.get(); const provider = choice?.provider ?? voiceProviderSetting.get();
return { return {
communicator: communicator?.getNativeCommunicator() ?? null, communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null, usePhoneMic: communicator === null,
provider, provider,
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
elevenLabsApiKey: elevenLabsApiKeySetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), openAiApiKey: openAiApiKeySetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
+5 -5
View File
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged, import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures' import type { PhoneGesture } from '../phone-ui/phone-gestures'
@@ -416,7 +416,6 @@ export class IosPreviewController {
const phone = readPhoneBatteryState() const phone = readPhoneBatteryState()
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
this.offSettings = onAnySettingChanged(() => { this.offSettings = onAnySettingChanged(() => {
this.syncContinuousCapture()
this.syncLockSetting() this.syncLockSetting()
this.pushBrightness() this.pushBrightness()
this.relayout() this.relayout()
@@ -1199,13 +1198,14 @@ export class IosPreviewController {
/** /**
* The transcription provider setting and the keys the cloud ones need. A * The transcription provider setting and the keys the cloud ones need. A
* window that needs its own provider and language (Translate) sets them for * window that needs its own provider and language (Translate) sets them for
* every capture while it is open; otherwise use the shared language preference. * every capture while it is open; otherwise only Whisper server is told a
* language (its own setting).
*/ */
private voiceProvider(): VoiceProviderSelection { private voiceProvider(): VoiceProviderSelection {
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
const provider = choice?.provider ?? voiceProviderSetting.get() const provider = choice?.provider ?? voiceProviderSetting.get()
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider), return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get() } whisperServerApiKey: whisperServerApiKeySetting.get() }
} }
+2 -32
View File
@@ -9,7 +9,7 @@ declare const java: any;
* than bundled in the APK; FaceclawVoiceController reads the files from the * than bundled in the APK; FaceclawVoiceController reads the files from the
* directory each model's `dirName` names. * directory each model's `dirName` names.
* *
* Downloadable models, same shape: * Two models, same shape:
* - "moonshine": the original on-device option (three files). The model is * - "moonshine": the original on-device option (three files). The model is
* no longer bundled in the APK; it is fetched into the same filesDir * no longer bundled in the APK; it is fetched into the same filesDir
* location that earlier releases copied the bundled files to, so * location that earlier releases copied the bundled files to, so
@@ -29,17 +29,12 @@ declare const java: any;
* used for Moonshine below. Every hash here was computed locally from a * used for Moonshine below. Every hash here was computed locally from a
* freshly downloaded copy of the file, not copied from a doc page. * freshly downloaded copy of the file, not copied from a doc page.
* *
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
* speech and auto-detection; English-only downloads remain untouched.
* SHA-256/size values come from the maintainer's LFS metadata, with the
* vocabulary hash computed locally. URLs pin the source revision.
*
* Mirrors the on-phone assistant model flow in llama.ts, except each model * Mirrors the on-phone assistant model flow in llama.ts, except each model
* here is multiple files rather than one; they download sequentially through * here is multiple files rather than one; they download sequentially through
* FaceclawModelDownloader (resume + pinned sha256 per file). * FaceclawModelDownloader (resume + pinned sha256 per file).
*/ */
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base"; export type AsrModelId = "moonshine" | "whisper-base-en";
type AsrModelFile = { type AsrModelFile = {
name: string; name: string;
@@ -110,30 +105,6 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
], ],
totalBytes: 160626066, totalBytes: 160626066,
}, },
"whisper-base": {
label: "Whisper (multilingual, base)",
dirName: "sherpa-onnx-whisper-base-int8",
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
files: [
{
name: "base-encoder.int8.onnx",
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
sizeBytes: 29120534,
},
{
name: "base-decoder.int8.onnx",
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
sizeBytes: 130672026,
},
{
name: "base-tokens.txt",
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
sizeBytes: 816730,
},
],
totalBytes: 160609290,
},
}; };
export type AsrModelState = { export type AsrModelState = {
@@ -158,7 +129,6 @@ function freshRuntime(): ModelRuntime {
const runtimes: Record<AsrModelId, ModelRuntime> = { const runtimes: Record<AsrModelId, ModelRuntime> = {
moonshine: freshRuntime(), moonshine: freshRuntime(),
"whisper-base-en": freshRuntime(), "whisper-base-en": freshRuntime(),
"whisper-base": freshRuntime(),
}; };
function modelDirPath(id: AsrModelId): string { function modelDirPath(id: AsrModelId): string {
+5 -8
View File
@@ -1,6 +1,6 @@
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt"; import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt"; import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt"; import { OpenAiRealtimeSttClient } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt"; import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt"; import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control"; import type { VoiceProviderKind } from "./voice-control";
@@ -14,12 +14,11 @@ export type VoiceProviderSelection = {
provider: VoiceProviderKind; provider: VoiceProviderKind;
/** /**
* Spoken language the provider should expect (ISO 639-1); empty lets it * Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Also used by on-device Whisper. * detect the language. Cloud providers only.
*/ */
language?: string; language?: string;
elevenLabsApiKey: string; elevenLabsApiKey: string;
openAiApiKey: string; openAiApiKey: string;
openAiTranscriptionModel?: OpenAiSttModel;
sonioxApiKey: string; sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */ /** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string; whisperServerUrl?: string;
@@ -53,7 +52,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
whisper: { whisper: {
name: "OpenAI", name: "OpenAI",
apiKey: (selection) => selection.openAiApiKey, apiKey: (selection) => selection.openAiApiKey,
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }), create: (options) => new OpenAiRealtimeSttClient(options),
}, },
"whisper-server": { "whisper-server": {
name: "Whisper server", name: "Whisper server",
@@ -77,10 +76,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
* What a capture transcribes with: the provider and the language it is told * What a capture transcribes with: the provider and the language it is told
* to expect. A running capture is shared only by holders asking for the same. * to expect. A running capture is shared only by holders asking for the same.
*/ */
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string { export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
const fallback = selection.provider === "onboard-whisper" ? "en" : ""; return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
} }
/** Whether createCloudSttClient would return a cloud client for this selection. */ /** Whether createCloudSttClient would return a cloud client for this selection. */
+17 -2
View File
@@ -2,7 +2,7 @@ import type { GrayImage } from '../graphics/image'
import { pebbleIcon } from '../graphics/pebble-icons' import { pebbleIcon } from '../graphics/pebble-icons'
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client' import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons' import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
import { rememberNotificationSources } from './notification-sources' import { rememberNotificationSources, shouldShowNotificationOnGlasses } from './notification-sources'
import type { AndroidNotification } from './notification-types' import type { AndroidNotification } from './notification-types'
export type { AndroidNotification, AndroidNotificationAction } from './notification-types' export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons' export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
@@ -24,8 +24,23 @@ export function onIosNotificationPopup(listener: (key: string) => void): () => v
export function iosNotificationMessage(): string { return active?.statusMessage ?? ANCS_CONNECT_MESSAGE } export function iosNotificationMessage(): string { return active?.statusMessage ?? ANCS_CONNECT_MESSAGE }
export function iosNotificationState(): AncsState { return active?.state ?? 'disconnected' } export function iosNotificationState(): AncsState { return active?.state ?? 'disconnected' }
export function readActiveNotifications(maxNotifications = 50) { return active?.read(maxNotifications) ?? [] } export function readActiveNotifications(maxNotifications = 50) { return active?.read(maxNotifications) ?? [] }
export function readVisibleNotifications(maxNotifications = 50) {
return readActiveNotifications(ALL_NOTIFICATIONS)
.filter(notification => shouldShowNotificationOnGlasses(notification.packageName))
.slice(0, Math.max(0, maxNotifications))
}
export function invokeNotificationAction(key: string, index: number): boolean { return active?.action(key,index) ?? false } export function invokeNotificationAction(key: string, index: number): boolean { return active?.action(key,index) ?? false }
export function dismissNotification(key: string): boolean { return active?.dismiss(key) ?? false } export function dismissNotification(key: string): boolean { return active?.dismiss(key) ?? false }
export function dismissAllNotifications(): boolean {
const client = active
if (!client) return false
// Snapshot: removals may arrive while dismissal requests are being sent.
let accepted = true
for (const notification of [...client.read(ALL_NOTIFICATIONS)]) {
if (!client.dismiss(notification.key)) accepted = false
}
return accepted
}
export function onAndroidNotificationPosted(listener: (key: string) => void): () => void { export function onAndroidNotificationPosted(listener: (key: string) => void): () => void {
listeners.add(listener); return () => { listeners.delete(listener) } listeners.add(listener); return () => { listeners.delete(listener) }
} }
@@ -44,7 +59,7 @@ function notificationIcon(notification: AndroidNotification): GrayImage {
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) { export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
// One icon per source, standing for its first notification. // One icon per source, standing for its first notification.
const sources = new Map<string, AndroidNotification>() const sources = new Map<string, AndroidNotification>()
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n) for (const n of readVisibleNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons)) const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false} return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
} }
+23 -2
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../graphics/image"; import { GrayImage } from "../graphics/image";
import { logCurrent, spanCurrent } from "./frame-timings"; import { logCurrent, spanCurrent } from "./frame-timings";
import { toUint8Array } from "../util/array-util"; import { toUint8Array } from "../util/array-util";
import { rememberNotificationSources } from "./notification-sources"; import { rememberNotificationSources, shouldShowNotificationOnGlasses } from "./notification-sources";
import { PHOTO_TONE, grayImageFromPacket } from "./image-files"; import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
import { NOTIFICATION_ICON_SIZE } from "./notification-types"; import { NOTIFICATION_ICON_SIZE } from "./notification-types";
@@ -21,6 +21,7 @@ const ICON_CACHE_MS = 60_000;
let cachedIcons: GrayImage[] = []; let cachedIcons: GrayImage[] = [];
let cachedKeys: string[] = []; let cachedKeys: string[] = [];
let cachedPackages = new Map<string, string>();
let cachedAtMs = 0; let cachedAtMs = 0;
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>(); const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
const KEYED_ICON_CACHE_MAX = 128; const KEYED_ICON_CACHE_MAX = 128;
@@ -95,6 +96,7 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
keys.push(index < keyList.size() ? String(keyList.get(index)) : ""); keys.push(index < keyList.size() ? String(keyList.get(index)) : "");
} }
cachedPackages = new Map(readActiveNotifications(ALL_NOTIFICATIONS).map(n => [n.key, n.packageName]));
cachedIcons = icons; cachedIcons = icons;
cachedKeys = keys; cachedKeys = keys;
cachedAtMs = now; cachedAtMs = now;
@@ -103,7 +105,12 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
/** Copies of the first maxIcons cached icons, with their keys. */ /** Copies of the first maxIcons cached icons, with their keys. */
function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult { function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult {
return { icons: cachedIcons.slice(0, maxIcons).map(icon => icon.clone()), keys: cachedKeys.slice(0, maxIcons), stale }; // Reapply preferences even to a warm/stale cache: toggles take effect immediately.
const indices = cachedKeys.map((_, index) => index).filter(index => {
const packageName = cachedPackages.get(cachedKeys[index]!);
return packageName !== undefined && shouldShowNotificationOnGlasses(packageName);
}).slice(0, maxIcons);
return { icons: indices.map(index => cachedIcons[index]!.clone()), keys: indices.map(index => cachedKeys[index]!), stale };
} }
export type NotificationIconResult = { export type NotificationIconResult = {
@@ -318,6 +325,13 @@ export function readActiveNotifications(maxNotifications = 50): AndroidNotificat
} }
} }
/** Glasses display only; discover all sources and filter before the display limit. */
export function readVisibleNotifications(maxNotifications = 50): AndroidNotification[] {
return readActiveNotifications(ALL_NOTIFICATIONS)
.filter(notification => shouldShowNotificationOnGlasses(notification.packageName))
.slice(0, Math.max(0, maxNotifications));
}
export function invokeNotificationAction(notificationKey: string, actionIndex: number): boolean { export function invokeNotificationAction(notificationKey: string, actionIndex: number): boolean {
if (!global.isAndroid || !notificationKey) return false; if (!global.isAndroid || !notificationKey) return false;
invalidateIconCaches(); invalidateIconCaches();
@@ -337,6 +351,13 @@ export function dismissNotification(notificationKey: string): boolean {
); );
} }
/** Explicit user action: dismiss on the phone as well as on the glasses. */
export function dismissAllNotifications(): boolean {
if (!global.isAndroid) return false;
invalidateIconCaches();
return Boolean(com.faceclaw.app.FaceclawMediaNotificationListenerService.dismissAllNotifications());
}
export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void { export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void {
return addNotificationListener(notificationPostedListeners, listener); return addNotificationListener(notificationPostedListeners, listener);
} }
+30 -5
View File
@@ -1,6 +1,16 @@
import { getStringSetting, setStringSetting } from "./settings-store"; import { getStringSetting, setStringSetting } from "./settings-store";
const SOURCES_KEY = "notifications.sources"; const SOURCES_KEY = "notifications.sources";
const DEFAULT_KEY = "notifications.newAppsEnabledByDefault";
/** Unset preserves the behavior of existing installations. */
export function newNotificationAppsEnabledByDefault(): boolean {
return getStringSetting(DEFAULT_KEY, "true") !== "false";
}
export function setNewNotificationAppsEnabledByDefault(enabled: boolean): void {
setStringSetting(DEFAULT_KEY, String(enabled));
}
export type NotificationSource = { export type NotificationSource = {
packageName: string; packageName: string;
@@ -8,8 +18,15 @@ export type NotificationSource = {
showOnGlasses: boolean; showOnGlasses: boolean;
}; };
type SavedSource = Omit<NotificationSource, "showOnGlasses"> & { showOnGlasses?: boolean };
/** Sources outlive their active notifications so a muted app can always be re-enabled. */ /** Sources outlive their active notifications so a muted app can always be re-enabled. */
export function readNotificationSources(): NotificationSource[] { export function readNotificationSources(): NotificationSource[] {
const enabled = newNotificationAppsEnabledByDefault();
return readSavedSources().map((source) => ({ ...source, showOnGlasses: source.showOnGlasses ?? enabled }));
}
function readSavedSources(): SavedSource[] {
try { try {
const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]")); const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]"));
if (!Array.isArray(parsed)) return []; if (!Array.isArray(parsed)) return [];
@@ -17,7 +34,7 @@ export function readNotificationSources(): NotificationSource[] {
.map((source) => ({ .map((source) => ({
packageName: source.packageName, packageName: source.packageName,
appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName, appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName,
showOnGlasses: source.showOnGlasses !== false, showOnGlasses: typeof source.showOnGlasses === "boolean" ? source.showOnGlasses : undefined,
})); }));
} catch { } catch {
return []; return [];
@@ -25,7 +42,7 @@ export function readNotificationSources(): NotificationSource[] {
} }
export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void { export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void {
const sources = new Map(readNotificationSources().map((source) => [source.packageName, source])); const sources = new Map(readSavedSources().map((source) => [source.packageName, source]));
let changed = false; let changed = false;
for (const notification of notifications) { for (const notification of notifications) {
if (!notification.packageName) continue; if (!notification.packageName) continue;
@@ -35,7 +52,7 @@ export function rememberNotificationSources(notifications: { packageName: string
sources.set(notification.packageName, { sources.set(notification.packageName, {
packageName: notification.packageName, packageName: notification.packageName,
appName, appName,
showOnGlasses: previous?.showOnGlasses ?? true, showOnGlasses: previous?.showOnGlasses,
}); });
changed = true; changed = true;
} }
@@ -44,12 +61,13 @@ export function rememberNotificationSources(notifications: { packageName: string
} }
export function shouldShowNotificationOnGlasses(packageName: string): boolean { export function shouldShowNotificationOnGlasses(packageName: string): boolean {
return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses ?? true; return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses
?? newNotificationAppsEnabledByDefault();
} }
export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void { export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void {
if (!source.packageName) return; if (!source.packageName) return;
const sources = new Map(readNotificationSources().map((item) => [item.packageName, item])); const sources = new Map(readSavedSources().map((item) => [item.packageName, item]));
sources.set(source.packageName, { sources.set(source.packageName, {
packageName: source.packageName, packageName: source.packageName,
appName: source.appName || source.packageName, appName: source.appName || source.packageName,
@@ -57,3 +75,10 @@ export function setNotificationSourceEnabled(source: { packageName: string; appN
}); });
setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values()))); setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
} }
/** Set explicit choices only for sources already discovered; leave the future default alone. */
export function setAllNotificationSourcesEnabled(enabled: boolean): void {
setStringSetting(SOURCES_KEY, JSON.stringify(readSavedSources().map((source) => ({
...source, showOnGlasses: enabled,
}))));
}
+7 -10
View File
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
/** /**
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session * OpenAI realtime speech-to-text over WebSocket (a transcription-only session
* on the Realtime API, with a configurable transcription model). * on the Realtime API, with the streaming gpt-realtime-whisper model).
* https://developers.openai.com/api/docs/guides/realtime-transcription * https://developers.openai.com/api/docs/guides/realtime-transcription
* *
* Like the ElevenLabs client, push-to-talk defines the utterance boundary: * Like the ElevenLabs client, push-to-talk defines the utterance boundary:
@@ -16,11 +16,10 @@ import { openSocket, type SocketConnection } from "./socket";
*/ */
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper"; const MODEL_ID = "gpt-realtime-whisper";
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
const TARGET_SAMPLE_RATE = 24000; const TARGET_SAMPLE_RATE = 24000;
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel }; export type OpenAiSttOptions = CloudSttOptions;
export class OpenAiRealtimeSttClient implements CloudSttClient { export class OpenAiRealtimeSttClient implements CloudSttClient {
private ws: SocketConnection | null = null; private ws: SocketConnection | null = null;
@@ -41,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
onOpen: () => { onOpen: () => {
if (this.closed) return; if (this.closed) return;
this.open = true; this.open = true;
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model))); this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
for (const chunk of this.pendingChunks.splice(0)) { for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk); this.sendChunk(chunk);
} }
@@ -166,9 +165,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`); this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
return; return;
} }
this.options.onError(message.error?.code === "model_not_found" this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
return; return;
default: default:
return; return;
@@ -204,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
} }
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object { function sessionConfig(language: string): object {
return { return {
type: "session.update", type: "session.update",
session: { session: {
@@ -212,7 +209,7 @@ function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_
audio: { audio: {
input: { input: {
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
transcription: { model, ...(language ? { language } : {}) }, transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
// Push-to-talk owns the utterance boundary; no server VAD. // Push-to-talk owns the utterance boundary; no server VAD.
turn_detection: null, turn_detection: null,
}, },
-24
View File
@@ -1,24 +0,0 @@
/** Languages supported by both Whisper backends, in picker order. */
export const TRANSCRIPTION_LANGUAGES = {
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
} as const;
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
export function resolveTranscriptionLanguage(
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
): string {
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
if (provider === "onboard") return "";
if (choice === "default") {
if (provider === "onboard-whisper") return "en";
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
}
return choice === "auto" ? "" : choice;
}
+2 -5
View File
@@ -1,4 +1,3 @@
import type { OpenAiSttModel } from "./openai-stt";
import { Utils } from "@nativescript/core"; import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
@@ -25,7 +24,7 @@ export type VoiceControlState = {
detail: string; detail: string;
}; };
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model, // "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model // see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
@@ -55,12 +54,11 @@ export type PushToTalkOptions = {
provider: VoiceProviderKind; provider: VoiceProviderKind;
elevenLabsApiKey: string; elevenLabsApiKey: string;
openAiApiKey: string; openAiApiKey: string;
openAiTranscriptionModel?: OpenAiSttModel;
sonioxApiKey: string; sonioxApiKey: string;
whisperServerUrl?: string; whisperServerUrl?: string;
whisperServerModel?: string; whisperServerModel?: string;
whisperServerApiKey?: string; whisperServerApiKey?: string;
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */ /** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string; language?: string;
saveRecording: boolean; saveRecording: boolean;
/** /**
@@ -398,7 +396,6 @@ export class FaceclawVoiceControlBridge {
// Which on-device model to load; a no-op setter for every provider except // Which on-device model to load; a no-op setter for every provider except
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
this.controller?.setOnboardLanguage(options.language ?? "en");
const captureId = this.nextNativeCaptureId++; const captureId = this.nextNativeCaptureId++;
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
this.controller?.start("onboard", captureId); this.controller?.start("onboard", captureId);
+2 -30
View File
@@ -1,4 +1,3 @@
import type { OpenAiSttModel } from "../native/openai-stt";
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts"; import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures"; import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve"; import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
@@ -19,7 +18,6 @@ import {
} from "~/assistant/models"; } from "~/assistant/models";
import { isLocalModelReady } from "../native/local-llm"; import { isLocalModelReady } from "../native/local-llm";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat"; import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth"; import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
@@ -1138,35 +1136,9 @@ export const whisperServerModelSetting = new ConfigSettingString({
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.", description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
}); });
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
id: "openai-transcription-model",
label: "OpenAI transcription model",
storageKey: "voice.openAiTranscriptionModel",
defaultValue: "gpt-realtime-whisper",
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
});
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
id: "voice-language",
label: "Transcription language",
storageKey: "voice.language",
defaultValue: "default",
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
formatValue: (value) => value === "default" ? "Provider default"
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
});
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
}
export const whisperServerLanguageSetting = new ConfigSettingString({ export const whisperServerLanguageSetting = new ConfigSettingString({
id: "whisper-server-language", id: "whisper-server-language",
label: "Whisper server language", label: "Whisper language",
storageKey: "voice.whisperServerLanguage", storageKey: "voice.whisperServerLanguage",
defaultValue: "", defaultValue: "",
editorTitle: "Language code, e.g. en or sk (empty: auto)", editorTitle: "Language code, e.g. en or sk (empty: auto)",
@@ -1174,7 +1146,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
formatValue: (value) => value || "Auto-detect", formatValue: (value) => value || "Auto-detect",
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."), validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
description: description:
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.", "Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
}); });
export const whisperServerApiKeySetting = new ConfigSettingString({ export const whisperServerApiKeySetting = new ConfigSettingString({
+1 -6
View File
@@ -63,7 +63,6 @@ import {
mapboxApiKeySetting, mapboxApiKeySetting,
mirrorTouchSetting, mirrorTouchSetting,
openAiApiKeySetting, openAiApiKeySetting,
openAiTranscriptionModelSetting,
previewColorSetting, previewColorSetting,
previewScalingSetting, previewScalingSetting,
phoneRotationSetting, phoneRotationSetting,
@@ -88,7 +87,6 @@ import {
toggleSettingMenuItem, toggleSettingMenuItem,
useMicControlSetting, useMicControlSetting,
voiceProviderSetting, voiceProviderSetting,
voiceLanguageSetting,
screenTimeoutSetting, screenTimeoutSetting,
wakeToLastAppSetting, wakeToLastAppSetting,
wakeWordActionSetting, wakeWordActionSetting,
@@ -145,11 +143,9 @@ function settingsSections(): SettingsSection[] {
items: [ items: [
enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceProviderSetting),
enumSettingMenuItem(voiceLanguageSetting),
...whisperServerMenuItems(), ...whisperServerMenuItems(),
asrModelMenuItem("moonshine"), asrModelMenuItem("moonshine"),
asrModelMenuItem("whisper-base-en"), asrModelMenuItem("whisper-base-en"),
asrModelMenuItem("whisper-base"),
], ],
}, },
{ {
@@ -288,7 +284,7 @@ function settingsSections(): SettingsSection[] {
toggleSettingMenuItem(showBleBandwidthSetting)] }; toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer. // No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
if (section.label === "Display") return { ...section, items: section.items.filter(item => if (section.label === "Display") return { ...section, items: section.items.filter(item =>
item.label !== screenTimeoutSetting.label) }; item.label !== screenTimeoutSetting.label) };
return section; return section;
@@ -531,7 +527,6 @@ function asrModelStatusText(id: AsrModelId): string {
*/ */
function whisperServerMenuItems(): MenuItem[] { function whisperServerMenuItems(): MenuItem[] {
return [ return [
enumSettingMenuItem(openAiTranscriptionModelSetting),
textSettingMenuItem(whisperServerUrlSetting), textSettingMenuItem(whisperServerUrlSetting),
textSettingMenuItem(whisperServerModelSetting), textSettingMenuItem(whisperServerModelSetting),
textSettingMenuItem(whisperServerLanguageSetting), textSettingMenuItem(whisperServerLanguageSetting),
+20 -8
View File
@@ -2,7 +2,7 @@ import { getDefaultSmallFont } from "../graphics/ui-fonts";
import { GrayImage } from "../graphics/image"; import { GrayImage } from "../graphics/image";
import { truncateText } from "../graphics/textwrap"; import { truncateText } from "../graphics/textwrap";
import { ALL_NOTIFICATIONS, readActiveNotifications } from "../native/notification-icons"; import { ALL_NOTIFICATIONS, readActiveNotifications } from "../native/notification-icons";
import { readNotificationSources, setNotificationSourceEnabled, shouldShowNotificationOnGlasses } from "../native/notification-sources"; import { newNotificationAppsEnabledByDefault, setNewNotificationAppsEnabledByDefault, setAllNotificationSourcesEnabled, readNotificationSources, setNotificationSourceEnabled, shouldShowNotificationOnGlasses } from "../native/notification-sources";
import { type InputEvent } from "./gestures"; import { type InputEvent } from "./gestures";
import { type Layer, type LayerContext, type PaintBelow } from "./layers"; import { type Layer, type LayerContext, type PaintBelow } from "./layers";
import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu"; import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu";
@@ -11,14 +11,26 @@ import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu";
export class NotificationFilterLayer implements Layer { export class NotificationFilterLayer implements Layer {
private menu: MenuLayer | null = null; private menu: MenuLayer | null = null;
private menuWidth = 0; private menuWidth = 0;
/** The package behind each current row, parallel to the menu's items. */ /** Stable row keys, including controls, parallel to the menu's items. */
private packages: string[] = []; private packages: string[] = [];
private currentMenu(ctx: LayerContext): MenuLayer { private currentMenu(ctx: LayerContext): MenuLayer {
readActiveNotifications(ALL_NOTIFICATIONS); readActiveNotifications(ALL_NOTIFICATIONS);
const sources = readNotificationSources().sort((a, b) => const sources = readNotificationSources().sort((a, b) =>
a.appName.localeCompare(b.appName) || a.packageName.localeCompare(b.packageName)); a.appName.localeCompare(b.appName) || a.packageName.localeCompare(b.packageName));
const items: MenuItem[] = sources.map((source) => ({ const items: MenuItem[] = [{
label: "New apps enabled by default",
onSelect: () => setNewNotificationAppsEnabledByDefault(!newNotificationAppsEnabledByDefault()),
render: ({ image, x, y, width, selected }) => {
const font = getDefaultSmallFont();
drawToggleMenuItem(image, font, x, y, width,
truncateText(font, "New apps enabled by default", width - 48), newNotificationAppsEnabledByDefault(), selected);
},
}, {
label: "Enable all", onSelect: () => setAllNotificationSourcesEnabled(true),
}, {
label: "Disable all", onSelect: () => setAllNotificationSourcesEnabled(false),
}, ...sources.map<MenuItem>((source) => ({
label: source.appName, label: source.appName,
onSelect: () => setNotificationSourceEnabled(source, !shouldShowNotificationOnGlasses(source.packageName)), onSelect: () => setNotificationSourceEnabled(source, !shouldShowNotificationOnGlasses(source.packageName)),
render: ({ image, x, y, width, selected }) => { render: ({ image, x, y, width, selected }) => {
@@ -26,11 +38,11 @@ export class NotificationFilterLayer implements Layer {
drawToggleMenuItem(image, font, x, y, width, truncateText(font, source.appName, width - 48), drawToggleMenuItem(image, font, x, y, width, truncateText(font, source.appName, width - 48),
shouldShowNotificationOnGlasses(source.packageName), selected); shouldShowNotificationOnGlasses(source.packageName), selected);
}, },
})); }))];
if (!items.length) items.push({ label: "No notification sources discovered yet.", disabled: true, onSelect: () => {} }); if (!sources.length) items.push({ label: "No notification sources discovered yet.", disabled: true, onSelect: () => {} });
const selectedPackage = this.menu?.selectedIndex != null ? this.packages[this.menu.selectedIndex] : undefined; const selectedPackage = this.menu?.selectedIndex != null ? this.packages[this.menu.selectedIndex] : undefined;
const index = sources.findIndex((source) => source.packageName === selectedPackage); this.packages = ["default", "enable-all", "disable-all", ...sources.map((source) => `source:${source.packageName}`)];
this.packages = sources.map((source) => source.packageName); const index = selectedPackage === undefined ? 0 : this.packages.indexOf(selectedPackage);
const width = ctx.stack.getBaseSize().width; const width = ctx.stack.getBaseSize().width;
if (!this.menu || width !== this.menuWidth) { if (!this.menu || width !== this.menuWidth) {
@@ -38,7 +50,7 @@ export class NotificationFilterLayer implements Layer {
this.menu = new MenuLayer("Notification filter", items, { this.menu = new MenuLayer("Notification filter", items, {
x: 8, y: 8, width: width - 16, x: 8, y: 8, width: width - 16,
opaque: true, showBorder: false, opaque: true, showBorder: false,
footer: "On = show popups on glasses", footer: "On = show notifications on glasses",
}).selectItem(Math.max(0, index)); }).selectItem(Math.max(0, index));
} else { } else {
this.menu.setItems(items, index >= 0 ? index : undefined); this.menu.setItems(items, index >= 0 ? index : undefined);
+8 -8
View File
@@ -8,7 +8,7 @@ import {
ALL_NOTIFICATIONS, ALL_NOTIFICATIONS,
dismissNotification, dismissNotification,
invokeNotificationAction, invokeNotificationAction,
readActiveNotifications, readVisibleNotifications,
readNotificationCustomView, readNotificationCustomView,
readNotificationIconByKey, readNotificationIconByKey,
readNotificationLargeIcon, readNotificationLargeIcon,
@@ -141,7 +141,7 @@ export class NotificationsListLayer implements Layer {
const image = new GrayImage(width, height, 0); const image = new GrayImage(width, height, 0);
const cardWidth = width - 2 * CARD_X; const cardWidth = width - 2 * CARD_X;
const cardTextWidth = cardWidth - CARD_TEXT_X - 14; const cardTextWidth = cardWidth - CARD_TEXT_X - 14;
const notifications = readActiveNotifications(MAX_NOTIFICATIONS); const notifications = readVisibleNotifications(MAX_NOTIFICATIONS);
const selectedIndex = this.resolveSelectedIndex(notifications); const selectedIndex = this.resolveSelectedIndex(notifications);
const layouts = notifications.map((notification, index) => const layouts = notifications.map((notification, index) =>
buildNotificationCardLayout(font, notification, index === selectedIndex, cardTextWidth), buildNotificationCardLayout(font, notification, index === selectedIndex, cardTextWidth),
@@ -185,7 +185,7 @@ export class NotificationsListLayer implements Layer {
} }
handleInput(event: InputEvent, ctx: LayerContext): void { handleInput(event: InputEvent, ctx: LayerContext): void {
const notifications = readActiveNotifications(MAX_NOTIFICATIONS); const notifications = readVisibleNotifications(MAX_NOTIFICATIONS);
const selectedIndex = this.resolveSelectedIndex(notifications); const selectedIndex = this.resolveSelectedIndex(notifications);
if (event.type === "double-click") { if (event.type === "double-click") {
// At the app's root this is intercepted by the yield wrapper; reached // At the app's root this is intercepted by the yield wrapper; reached
@@ -252,7 +252,7 @@ export class SingleNotificationLayer implements Layer {
const font = getDefaultSmallFont(); const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize(); const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0); const image = new GrayImage(width, height, 0);
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey); const notification = readVisibleNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
if (!notification) { if (!notification) {
return this.closeUnavailableNotification(ctx, paintBelow); return this.closeUnavailableNotification(ctx, paintBelow);
@@ -273,7 +273,7 @@ export class SingleNotificationLayer implements Layer {
else await menu.handleInput(event); else await menu.handleInput(event);
return; return;
} }
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey); const notification = readVisibleNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
if (!notification) { if (!notification) {
this.closeUnavailableNotification(ctx); this.closeUnavailableNotification(ctx);
return; return;
@@ -297,7 +297,7 @@ export class SingleNotificationLayer implements Layer {
this.close(ctx); this.close(ctx);
} else if (item.kind === "action") { } else if (item.kind === "action") {
invokeNotificationAction(this.notificationKey, item.action.index); invokeNotificationAction(this.notificationKey, item.action.index);
if (!readActiveNotifications(ALL_NOTIFICATIONS).some((active) => active.key === this.notificationKey)) { if (!readVisibleNotifications(ALL_NOTIFICATIONS).some((active) => active.key === this.notificationKey)) {
this.closeUnavailableNotification(ctx); this.closeUnavailableNotification(ctx);
} }
} else if (item.kind === "dismiss") { } else if (item.kind === "dismiss") {
@@ -310,7 +310,7 @@ export class SingleNotificationLayer implements Layer {
menu: new Menu<ConfirmationItem>({ menu: new Menu<ConfirmationItem>({
items: [ items: [
{ label: "Cancel", run: (runCtx) => this.close(runCtx) }, { label: "Cancel", run: (runCtx) => this.close(runCtx) },
{ label: "Turn off popups", run: (runCtx) => { { label: "Hide on glasses", run: (runCtx) => {
setNotificationSourceEnabled(notification, false); setNotificationSourceEnabled(notification, false);
this.close(runCtx); this.close(runCtx);
} }, } },
@@ -341,7 +341,7 @@ export class SingleNotificationLayer implements Layer {
}; };
menu.paint(image, menuBox, ctx.stack.isFocused()); menu.paint(image, menuBox, ctx.stack.isFocused());
const top = menuBox.y + menuBox.height + CONFIRM_PROMPT_GAP; const top = menuBox.y + menuBox.height + CONFIRM_PROMPT_GAP;
const prompt = `Turn off notification popups from ${source.appName || source.packageName}? You can turn them back on in the Notifications app's Notification filter.`; const prompt = `Hide notifications on glasses from ${source.appName || source.packageName}? You can turn them back on in the Notifications app's Notification filter.`;
const lines = wrapText(font, prompt, width - 2 * CONFIRM_X); const lines = wrapText(font, prompt, width - 2 * CONFIRM_X);
const maxLines = Math.max(1, Math.floor((height - top - 12) / lineStep(font))); const maxLines = Math.max(1, Math.floor((height - top - 12) / lineStep(font)));
for (let index = 0; index < Math.min(lines.length, maxLines); index++) { for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
@@ -0,0 +1,20 @@
# Dismiss all notifications
The Notifications app menu offers Dismiss all notifications, disabled when the
list is empty. This explicit action dismisses notifications on the phone too;
popup filtering and source/default preferences remain independent.
Android uses NotificationListenerService.cancelAllNotifications, so dismissal
is not limited to the 50 items displayed by the list and Android controls which
notifications are removable. The normal removal callbacks update the list and
popups. Missing listener access returns false without changing preferences.
iOS uses the existing ANCS dismissal operation on a snapshot of all known keys;
notifications without a negative action may remain.
Software tests cover menu action/disabled state, more than 50 notifications,
native bridge dispatch, disconnected access and ANCS bulk dismissal.
On a device, verify the permission-granted/revoked cases, protected or ongoing
notifications, notification groups, and consistency with the phone's tray.
Hardware validation remains pending.
Reference: https://developer.android.com/reference/android/service/notification/NotificationListenerService#cancelAllNotifications()
+35
View File
@@ -0,0 +1,35 @@
# Notification filter validation
Software: `npm test` and `bash lint.sh app/native/notification-sources.ts app/ui/notification-filter.ts`.
The notification suite covers legacy preferences, reloads of the settings store,
discovery, default changes without overwriting exceptions, both bulk actions,
empty lists, stable menu selection and popup suppression before wake.
Hardware validation (pending; no glasses tested):
1. On an existing installation, check that enabled and muted applications retain
their behavior before touching the new default.
2. Turn “New apps enabled by default” off; restart the app. Send a notification
from a previously unknown package: it must appear in the filter and on the
phone, without waking the glasses or showing a glasses popup.
3. Enable that package individually; send another notification and check its popup.
4. Change the default in both directions; verify individual exceptions survive.
5. Use “Disable all”, then discover another package: it follows the default.
Repeat with “Enable all” while the default is off.
6. Restart and repeat notifications from enabled and disabled sources. Check that
bulk actions and filtering never dismiss notifications on the phone.
The same source preferences now filter glasses popups, the Notifications list,
and the notification tray/status icons. Filtering happens before display limits;
source discovery and the phone notification lifecycle remain unfiltered.
Cached tray icons reapply the current preferences on each read, including stale
reads, so a changed toggle cannot leave a muted icon visible.
Additional software coverage: Android list limits and warm/stale icon caches,
Disable all / re-enable without a new post, and iOS list/tray filtering.
Additional hardware validation (pending): with mixed enabled/disabled sources,
check the list and status bar together. Toggle a source while its notification
is already active, then Disable all and Enable all. Verify immediate removal /
restoration without a new notification, including after restart, and verify
notifications remain available in the phone notification shade.
-68
View File
@@ -1,68 +0,0 @@
# Whisper transcription languages
Settings > Voice > Transcription language selects French, English, 27 other
languages, or auto-detection. It applies to ordinary dictation and continuous
transcription using OpenAI, on-device Whisper, and the other cloud providers.
Translate retains its own spoken-language preference. Moonshine remains
English-only; Apple's on-device speech recognizer retains its own locale.
The initial **Provider default** preserves existing installations: English for
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
Whisper server language for self-hosted Whisper. Selecting another shared
language overrides the server language without deleting it. Returning to Provider
default restores it. Settings persist through the existing settings store.
For French or auto-detection with **On-device (Whisper)**, download
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
English uses the existing English-only base model; neither model is deleted or
downloaded automatically. A missing model uses the existing “Voice model not
downloaded” status. Language changes restart active continuous transcription;
other captures use the new language when started.
## Software validation
- JavaScript regression tests exercise stored settings/reload, compatible
defaults, French and auto-detection, Translate overrides in Android/iOS capture
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
- Kotlin tests exercise English/French/other/automatic model selection and file
names. Android release compilation checks the actual sherpa-onnx bridge.
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
the existing resumable downloader. Encoder/decoder hashes and sizes come from
repository LFS metadata; the vocabulary hash was computed from its contents.
## Device validation still required
With both phone microphone preview and glasses microphone, check French dictation
(including accents), auto-detection, English rollback, continuous capture while
changing language, cancellation/resumption of model download, and persistence
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
No paid API call, microphone recording, or firmware update is needed for the
software checks above; recognition accuracy and latency require device testing.
## References
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
## OpenAI transcription model selection
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
The unset default retains gpt-realtime-whisper for existing installations; no
provider, language or Assistant model preference is overwritten. This setting
applies to Translate's speech recognition as well as Transcribe and dictation.
The two transcribe alternatives produce transcript deltas after a committed
audio segment; existing pause segmentation and push-to-talk release supply the
commits. Choosing a model retargets active continuous capture.
Model access errors name the requested model before the long project error and
point to the setting. Factory/session-wire tests cover model, language, manual
commit and final transcript behavior; settings tests cover persistence and the
Android/iOS option plumbing. No real API inference was performed; actual project
access, recognition and latency remain to be checked on a device. The model list
provided by the operator establishes account visibility, not a successful audio
request.
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
@@ -1,18 +0,0 @@
package com.faceclaw.app
/** The English-only files remain usable after upgrading; other languages need multilingual base. */
data class WhisperModelFiles(val language: String) {
val englishOnly: Boolean get() = language == "en"
val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
private val prefix: String get() = if (englishOnly) "base.en" else "base"
val encoder: String get() = "$prefix-encoder.int8.onnx"
val decoder: String get() = "$prefix-decoder.int8.onnx"
val tokens: String get() = "$prefix-tokens.txt"
val files: List<String> get() = listOf(encoder, decoder, tokens)
companion object {
/** Null preserves the old English default; an empty code enables automatic detection. */
fun forLanguage(language: String?): WhisperModelFiles =
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
}
}
@@ -55,7 +55,6 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
ASR_MODELS: { ASR_MODELS: {
moonshine: { label: 'Moonshine', sizeBytes: 1 }, moonshine: { label: 'Moonshine', sizeBytes: 1 },
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 }, 'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
}, },
asrModelState: () => ({}), cancelAsrModelDownload() {}, asrModelState: () => ({}), cancelAsrModelDownload() {},
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {}, deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
-1
View File
@@ -26,7 +26,6 @@ function settingsFixture(store = new Map()) {
}; };
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
}}; }};
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
+2 -44
View File
@@ -207,8 +207,6 @@ test('provider selection falls back to on-device without a key and connects the
// A running capture is shared only on the same provider and language. // A running capture is shared only on the same provider and language.
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja'); assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
const statuses = []; const statuses = [];
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
@@ -227,13 +225,13 @@ test('provider selection falls back to on-device without a key and connects the
}); });
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => { test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
for (const language of ['fr', 'ja', '']) { for (const language of ['ja', '']) {
const elevenlabs = harness('elevenlabs', true, { language }); const elevenlabs = harness('elevenlabs', true, { language });
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null); assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
const soniox = harness('soniox', true, { language }); const soniox = harness('soniox', true, { language });
soniox.open(); soniox.open();
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined); assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
const openai = harness('openai', true, { language }); const openai = harness('openai', true, { language });
openai.open(); openai.open();
@@ -242,43 +240,3 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
assert.equal(transcription.language, language || undefined); assert.equal(transcription.language, language || undefined);
} }
}); });
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
const h = harness('openai', true, { model, language: 'fr' });
h.open();
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
assert.equal(input.transcription.model, model);
assert.equal(input.transcription.language, 'fr');
assert.equal(input.turn_detection, null);
h.client.acceptPcm(pcm(1200, 150));
h.client.acceptPcm(pcm(0, 1500));
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
assert.equal(h.events.at(-1).isFinal, true);
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
assert.ok(h.errors[0].includes(model));
assert.ok(h.errors[0].includes('OpenAI transcription model'));
}
});
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
const created = [];
const load = loader({ setTimeout, clearTimeout }, {
'./openai-stt': {
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
},
});
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
assert.equal(created[0].model, selection.openAiTranscriptionModel);
assert.equal(created[0].language, 'fr');
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
});
+69
View File
@@ -0,0 +1,69 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
test('Notifications menu dismisses all, including beyond the visible list, and leaves filters untouched', () => {
let options, active = Array.from({ length: 80 }, (_, i) => ({ key: String(i) }));
let dismissals = 0, renders = 0, pops = 0;
const load = loader({}, {
'../../ui/notifications': { NotificationsListLayer: class {} },
'../../ui/notification-filter': { NotificationFilterLayer: class {} },
'../../native/notification-icons': {
ALL_NOTIFICATIONS: 0x7fffffff,
readActiveNotifications(max) { assert.equal(max, 0x7fffffff); return active; },
dismissAllNotifications() { dismissals++; active = []; return true; },
onAndroidNotificationsChanged: () => () => {},
},
'../../ui/shell/in-process-window': {
YieldAtRootLayer: class {},
createInProcessWindow(value) { options = value; return { requestRender() { renders++; } }; },
},
});
load('app/apps/notifications/notifications-app.ts').createNotificationsAppWindow({});
const items = options.menuItems();
assert.equal(items[0].label, 'Notification filter');
const action = items.find(x => x.label === 'Dismiss all notifications');
assert.equal(action.disabled(), false);
assert.equal(dismissals, 0);
action.onSelect({ stack: { pop() { pops++; } } });
assert.equal(dismissals, 1);
assert.equal(pops, 1);
assert.equal(renders, 1);
assert.equal(action.disabled(), true);
});
test('Android bulk action uses the native listener and propagates unavailable access', () => {
let calls = 0, allowed = false;
const load = loader({ global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawMediaNotificationListenerService: { dismissAllNotifications() { calls++; return allowed; } },
} } } }, {
'../graphics/image': {}, './frame-timings': {}, '../util/array-util': {},
'./notification-sources': {}, './image-files': { PHOTO_TONE: { gamma: 1 } }, './notification-types': { NOTIFICATION_ICON_SIZE: 24 },
});
const api = load('app/native/notification-icons.ts');
assert.equal(api.dismissAllNotifications(), false);
allowed = true;
assert.equal(api.dismissAllNotifications(), true);
assert.equal(calls, 2);
});
test('iOS snapshots all ANCS keys and continues when a notification cannot be dismissed', () => {
const load = loader({}, {
'../graphics/pebble-icons': {}, '../g2/ancs-client': {}, './notification-app-icons': {}, './notification-sources': {},
});
const api = load('app/native/notification-icons.ios.ts');
assert.equal(api.dismissAllNotifications(), false);
const active = Array.from({ length: 80 }, (_, i) => ({ key: String(i) })), attempted = [];
api.bindIosNotifications({
read(max) { assert.equal(max, api.ALL_NOTIFICATIONS); return active; },
dismiss(key) {
attempted.push(key);
if (key === '1') return false;
active.splice(active.findIndex(x => x.key === key), 1);
return true;
},
});
assert.equal(api.dismissAllNotifications(), false);
assert.equal(attempted.length, 80);
assert.deepEqual(active, [{ key: '1' }]);
});
-1
View File
@@ -12,7 +12,6 @@ function settingsFixture(store = new Map()) {
}; };
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js'); if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js'); if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }; return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
}}; }};
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), { vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
+2 -4
View File
@@ -6,8 +6,7 @@ const ts = require('typescript');
const path = require('node:path'); const path = require('node:path');
// The provider settings ios-preview-controller hands the voice bridge. // The provider settings ios-preview-controller hands the voice bridge.
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' }, const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
getVoiceTranscriptionLanguage: () => '', openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' }, whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } }; whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
function load(file, context) { function load(file, context) {
@@ -169,7 +168,6 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
let cachedFont = 'Light'; let cachedFont = 'Light';
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true }, const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) } require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] }, : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
setTimeout: fn => tasks.push(fn), setTimeout: fn => tasks.push(fn),
}); });
@@ -386,7 +384,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
recordTo: name => { bridge.recording = name; }, recordTo: name => { bridge.recording = name; },
setRawSession: session => { bridge.rawSession = session; } }; setRawSession: session => { bridge.rawSession = session; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' }, const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en', whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
saveTranscribeAudioSetting: { get: () => true } }; saveTranscribeAudioSetting: { get: () => true } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
+9 -1
View File
@@ -9,7 +9,7 @@ const generator = require('../scripts/generate-pebble-icons.cjs');
function setup(notifications) { function setup(notifications) {
const load = loader({}, { const load = loader({}, {
'../g2/ancs-client': { AncsClient: class {}, ANCS_CONNECT_MESSAGE: '' }, '../g2/ancs-client': { AncsClient: class {}, ANCS_CONNECT_MESSAGE: '' },
'./notification-sources': { rememberNotificationSources() {} }, './notification-sources': { rememberNotificationSources() {}, shouldShowNotificationOnGlasses: packageName => packageName !== "muted" },
}); });
const icons = load('app/native/notification-icons.ios.ts'); const icons = load('app/native/notification-icons.ios.ts');
icons.bindIosNotifications({ read: () => notifications }); icons.bindIosNotifications({ read: () => notifications });
@@ -106,3 +106,11 @@ test('the icon generator inverts Pebble art: black ink lights up, white fills hi
const decoded = loader({}, { './pebble-icons-data': loader({}, {})(dataFile) })('app/graphics/pebble-icons.ts').pebbleIcon('probe'); const decoded = loader({}, { './pebble-icons-data': loader({}, {})(dataFile) })('app/graphics/pebble-icons.ts').pebbleIcon('probe');
assert.deepEqual([...decoded.pixels], [...pixels]); assert.deepEqual([...decoded.pixels], [...pixels]);
}); });
test('muted sources are hidden before list and icon limits without removing phone notifications', () => {
const notifications = [notification('hidden', 'muted', '0'), notification('shown', 'allowed', '0')];
const { icons } = setup(notifications);
assert.deepEqual([...icons.readVisibleNotifications(1)].map(n => n.key), ['shown']);
assert.deepEqual([...icons.readActiveNotificationIcons(1, false).keys], ['shown']);
assert.equal(icons.readActiveNotifications(10).length, 2);
});
@@ -1,35 +0,0 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class WhisperModelFilesTest {
@Test fun existingEnglishFilesRemainUsable() {
for (language in listOf(null, "en", " EN ")) {
val model = WhisperModelFiles.forLanguage(language)
assertTrue(model.englishOnly)
assertEquals("en", model.language)
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
}
}
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
for (language in listOf("fr", "de", "ja", " FR ")) {
val model = WhisperModelFiles.forLanguage(language)
assertFalse(model.englishOnly)
assertEquals(language.trim().lowercase(), model.language)
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
}
}
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
val model = WhisperModelFiles.forLanguage("")
assertFalse(model.englishOnly)
assertEquals("", model.language)
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
}
}
+94
View File
@@ -70,6 +70,7 @@ function ui(fontSize = 12) {
const native = { const native = {
ALL_NOTIFICATIONS: 0x7fffffff, ALL_NOTIFICATIONS: 0x7fffffff,
readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; }, readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; },
readVisibleNotifications: () => { prefs.rememberNotificationSources(active); return active.filter(n => prefs.shouldShowNotificationOnGlasses(n.packageName)); },
dismissNotification: () => { dismissals++; active = []; return true; }, dismissNotification: () => { dismissals++; active = []; return true; },
readNotificationIconByKey: () => ({ icon: null, stale: false }), readNotificationIconByKey: () => ({ icon: null, stale: false }),
}; };
@@ -165,6 +166,7 @@ test('cancel and back dismiss the popup without changing source preferences', as
test('filter toggles persist after notifications disappear and discovers new sources while open', async () => { test('filter toggles persist after notifications disappear and discovers new sources while open', async () => {
const app = ui(); const app = ui();
app.paint(app.filter); app.paint(app.filter);
for (let i = 0; i < 3; i++) await app.input(app.filter, 'scroll-down');
await app.input(app.filter, 'click'); await app.input(app.filter, 'click');
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false); assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
app.active([{ packageName: 'com.alpha', appName: 'Alpha' }]); app.active([{ packageName: 'com.alpha', appName: 'Alpha' }]);
@@ -177,6 +179,68 @@ test('filter toggles persist after notifications disappear and discovers new sou
assert.equal(app.prefs.shouldShowNotificationOnGlasses('com.alpha'), true); assert.equal(app.prefs.shouldShowNotificationOnGlasses('com.alpha'), true);
}); });
test('default persists without overwriting explicit choices, including legacy saved sources', () => {
const settings = new Map([['notifications.sources', JSON.stringify([
{ ...mail, showOnGlasses: true },
{ packageName: 'muted', appName: 'Muted', showOnGlasses: false },
])]]);
let prefs = store(settings);
prefs.rememberNotificationSources([{ packageName: 'discovered', appName: 'Discovered' }]);
prefs.setNewNotificationAppsEnabledByDefault(false);
prefs = store(settings);
assert.equal(prefs.newNotificationAppsEnabledByDefault(), false);
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
assert.equal(prefs.shouldShowNotificationOnGlasses('muted'), false);
assert.equal(prefs.shouldShowNotificationOnGlasses('discovered'), false);
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
prefs.rememberNotificationSources([{ packageName: 'future', appName: 'Future' }]);
prefs.setNotificationSourceEnabled({ packageName: 'future', appName: 'Future' }, true);
prefs.setNewNotificationAppsEnabledByDefault(true);
prefs.setNewNotificationAppsEnabledByDefault(false);
assert.equal(store(settings).shouldShowNotificationOnGlasses('future'), true);
});
test('bulk choices persist for known sources without changing the default for future sources', () => {
const settings = new Map();
let prefs = store(settings);
prefs.rememberNotificationSources([mail]);
prefs.setAllNotificationSourcesEnabled(false);
prefs = store(settings);
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), true);
prefs.setNewNotificationAppsEnabledByDefault(false);
prefs.setAllNotificationSourcesEnabled(true);
prefs = store(settings);
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
prefs.rememberNotificationSources([{ packageName: 'future', appName: 'Future' }]);
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
prefs.setAllNotificationSourcesEnabled(false);
assert.ok(prefs.readNotificationSources().every(source => !source.showOnGlasses));
});
test('filter controls work with an empty list and keep selection as sources arrive', async () => {
const app = ui();
app.active([]);
const labels = app.paint(app.filter).texts.map(({ text }) => text);
assert.ok(labels.includes('New apps enabled by default'));
assert.ok(labels.includes('Enable all'));
assert.ok(labels.includes('Disable all'));
await app.input(app.filter, 'click');
assert.equal(app.prefs.newNotificationAppsEnabledByDefault(), false);
app.active([mail]);
app.paint(app.filter);
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
await app.input(app.filter, 'scroll-down');
await app.input(app.filter, 'click');
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
assert.equal(app.prefs.newNotificationAppsEnabledByDefault(), false);
await app.input(app.filter, 'scroll-down');
await app.input(app.filter, 'click');
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
assert.equal(app.dismissals(), 0);
});
test('popup action wraps fully and stays visible with small and large fonts', async () => { test('popup action wraps fully and stays visible with small and large fonts', async () => {
for (const size of [12, 20]) { for (const size of [12, 20]) {
const app = ui(size); const app = ui(size);
@@ -254,3 +318,33 @@ test('controller filters before waking or opening a popup, and still refreshes t
elapse(); elapse();
assert.equal(popups, 1); assert.equal(popups, 1);
}); });
test('Android display filters before limits and reapplies preferences to cached tray icons', () => {
const prefs = store();
const active = [{ ...mail, key: 'hidden' }, { packageName: 'allowed', appName: 'Allowed', key: 'shown' }];
const file = ts.createSourceFile('icons.ts', source('app/native/notification-icons.ts'), ts.ScriptTarget.Latest, true);
const functions = file.statements.filter(node => ts.isFunctionDeclaration(node) &&
['readVisibleNotifications', 'cachedIconsResult'].includes(node.name?.text)).map(node => node.getText(file));
const { readVisibleNotifications, cachedIconsResult } = evaluate(functions.join('\n') + '\nexport { cachedIconsResult };', null, {
ALL_NOTIFICATIONS: 0x7fffffff,
readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; },
shouldShowNotificationOnGlasses: prefs.shouldShowNotificationOnGlasses,
cachedKeys: ['hidden', 'shown'],
cachedPackages: new Map(active.map(n => [n.key, n.packageName])),
cachedIcons: [{ clone: () => 'mail-icon' }, { clone: () => 'allowed-icon' }],
});
prefs.setNotificationSourceEnabled(mail, false);
assert.deepEqual(Array.from(readVisibleNotifications(1), n => n.key), ['shown']);
for (const stale of [false, true]) {
const result = cachedIconsResult(1, stale);
assert.deepEqual(Array.from(result.keys), ['shown']);
assert.deepEqual(Array.from(result.icons), ['allowed-icon']);
assert.equal(result.stale, stale);
}
prefs.setAllNotificationSourcesEnabled(false);
assert.equal(cachedIconsResult(10, false).keys.length, 0);
assert.equal(readVisibleNotifications(50).length, 0);
prefs.setNotificationSourceEnabled(mail, true);
assert.deepEqual(Array.from(cachedIconsResult(1, false).keys), ['hidden']);
assert.equal(active.length, 2);
});
-1
View File
@@ -28,7 +28,6 @@
"../app/ui/menu-core.ts", "../app/ui/menu-core.ts",
"../app/phone-ui/phone-gestures.ts", "../app/phone-ui/phone-gestures.ts",
"../app/native/java-direct-buffer.ts", "../app/native/java-direct-buffer.ts",
"../app/native/transcription-language.ts",
"../app/native/notification-types.ts", "../app/native/notification-types.ts",
"../app/g2/firmware-compat.ts", "../app/g2/firmware-compat.ts",
"../app/g2/glasses-hardware-identity.ts", "../app/g2/glasses-hardware-identity.ts",
+2 -24
View File
@@ -163,8 +163,7 @@ function androidBridge() {
starts: 0, starts: 0,
setListener(listener) { this.listener = listener; }, setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; }, stop() { this.active = false; },
isCapturing() { return this.active; }, isCapturing() { return this.active; },
@@ -172,8 +171,7 @@ function androidBridge() {
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null, './cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
}, { global: { isAndroid: true }, com: { faceclaw: { app: { }, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: function () { return controller; }, FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; }, FaceclawVoiceControllerListener: function (listener) { return listener; },
@@ -181,26 +179,6 @@ function androidBridge() {
return { bridge: new FaceclawVoiceControlBridge(), controller }; return { bridge: new FaceclawVoiceControlBridge(), controller };
} }
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
const { bridge, controller } = androidBridge();
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
assert.equal(controller.modelKind, 'whisper');
assert.equal(controller.language, 'fr');
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
assert.equal(controller.starts, 1);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
assert.equal(controller.language, '');
assert.equal(controller.starts, 2);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
// An omitted language preserves the native bridge's original English behavior.
assert.equal(controller.language, 'en');
assert.equal(controller.starts, 3);
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
assert.equal(controller.language, 'en');
assert.equal(controller.starts, 3);
bridge.stop();
});
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => { test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
const { bridge, controller } = androidBridge(); const { bridge, controller } = androidBridge();
const options = { provider: 'onboard-whisper' }; const options = { provider: 'onboard-whisper' };
-146
View File
@@ -1,146 +0,0 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
function load(file, requireModule = () => ({}), globals = {}) {
const exports = {};
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
return exports;
}
const languages = load('app/native/transcription-language.ts');
function settings(values = new Map()) {
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
: name.includes('settings-store') ? {
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
setStringSetting: (key, value) => values.set(key, value),
getBooleanSetting: (_key, fallback) => fallback,
onSettingsStoreChanged() {},
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
}
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
const s = settings(values);
assert.equal(s.voiceLanguageSetting.get(), 'default');
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
});
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
let s = settings(values);
s.voiceLanguageSetting.set('fr');
s = settings(values);
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
}
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
s.voiceLanguageSetting.set('auto');
s = settings(values);
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
s.voiceLanguageSetting.set('default');
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
});
test('corrupt language preferences revert to the compatible provider defaults', () => {
const s = settings(new Map([['voice.language', 'invalid']]));
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
});
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
const s = settings();
s.voiceLanguageSetting.set('fr');
s.voiceProviderSetting.set('whisper');
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
for (const [file, name, method] of [
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
]) {
const host = { continuousClients: new Map() };
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
// Run each real capture-options method with only its native dependencies stubbed.
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
const member = declaration.members.find(n => n.name?.getText(tree) === method);
const capture = loadBody(member.body.getText(tree), s, host);
assert.equal(capture().language, 'fr');
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
assert.equal(capture().provider, 'soniox');
assert.equal(capture().language, 'ja');
host.continuousClients.clear();
assert.equal(capture().language, 'fr');
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
}
});
function loadBody(body, s, host) {
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
}
test('multilingual downloads and deletion leave an existing English model intact', () => {
const files = new Map(), downloads = [];
const asr = load('app/native/asr-model.ts', () => ({
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
}), {
java: { io: { File: class {
constructor(name) { this.name = name; }
exists() { return files.has(this.name); }
length() { return files.get(this.name) ?? 0; }
delete() { return files.delete(this.name); }
} } },
com: { faceclaw: { app: {
FaceclawModelDownloaderListener: function(listener) { return listener; },
FaceclawModelDownloader: class {
constructor(url, destination, hash, size, listener) {
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
}
start() {}
cancel() {}
},
} } },
});
const english = asr.ASR_MODELS['whisper-base-en'];
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
assert.equal(asr.isAsrModelReady('whisper-base'), false);
asr.startAsrModelDownload('whisper-base');
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
for (let index = 0; index < 3; index++) {
const download = downloads[index];
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
assert.match(download.hash, /^[a-f0-9]{64}$/);
files.set(download.destination, download.size);
download.listener.onDone();
}
assert.equal(downloads.length, 3);
assert.equal(asr.isAsrModelReady('whisper-base'), true);
asr.deleteAsrModel('whisper-base');
assert.equal(asr.isAsrModelReady('whisper-base'), false);
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
});
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
const values = new Map();
let s = settings(values);
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
s.voiceLanguageSetting.set('fr');
s = settings(values);
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
assert.equal(s.voiceLanguageSetting.get(), 'fr');
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
});