forked from Blendman974/faceclaw
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f62573b1c
|
||
|
|
1959e57d02
|
No files matched your search
@@ -25,21 +25,18 @@ internal object AndroidSpeechEngines {
|
|||||||
private const val FEATURE_DIM = 80
|
private const val FEATURE_DIM = 80
|
||||||
|
|
||||||
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
||||||
fun recognizerConfig(
|
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
|
||||||
modelDir: File, kind: VoiceModelKind,
|
|
||||||
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
|
|
||||||
): OfflineRecognizerConfig {
|
|
||||||
val modelConfig = OfflineModelConfig.builder()
|
val modelConfig = OfflineModelConfig.builder()
|
||||||
.setNumThreads(1)
|
.setNumThreads(1)
|
||||||
if (kind == VoiceModelKind.WHISPER) {
|
if (kind == VoiceModelKind.WHISPER) {
|
||||||
modelConfig
|
modelConfig
|
||||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||||
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
|
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
|
||||||
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
|
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
|
||||||
.setLanguage(whisper.language)
|
.setLanguage("en")
|
||||||
.setTask("transcribe")
|
.setTask("transcribe")
|
||||||
.build())
|
.build())
|
||||||
.setTokens(File(modelDir, whisper.tokens).absolutePath)
|
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
|
||||||
} else {
|
} else {
|
||||||
modelConfig
|
modelConfig
|
||||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||||
|
|||||||
+12
@@ -504,6 +504,18 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@JvmStatic
|
||||||
|
fun dismissAllNotifications(): Boolean {
|
||||||
|
val service = activeService ?: return false
|
||||||
|
return try {
|
||||||
|
service.cancelAllNotifications()
|
||||||
|
true
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
Log.w(TAG, "failed to dismiss all notifications", t)
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun emitNotificationPosted(key: String?) {
|
private fun emitNotificationPosted(key: String?) {
|
||||||
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
|
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,17 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
"decoder_model_merged.ort",
|
"decoder_model_merged.ort",
|
||||||
"tokens.txt"
|
"tokens.txt"
|
||||||
)
|
)
|
||||||
|
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||||
|
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||||
|
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||||
|
// Whisper re-encodes the whole buffer on every call, so the shared session
|
||||||
|
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
|
||||||
|
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
|
||||||
|
private val ASR_WHISPER_MODEL_FILES = arrayOf(
|
||||||
|
"base.en-encoder.int8.onnx",
|
||||||
|
"base.en-decoder.int8.onnx",
|
||||||
|
"base.en-tokens.txt"
|
||||||
|
)
|
||||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||||
// (endpointing, transcript pacing) is tuned for.
|
// (endpointing, transcript pacing) is tuned for.
|
||||||
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
||||||
@@ -73,7 +84,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
private val mainHandler = Handler(Looper.getMainLooper())
|
private val mainHandler = Handler(Looper.getMainLooper())
|
||||||
@Volatile
|
@Volatile
|
||||||
private var communicator: FaceclawBleCommunicator? = null
|
private var communicator: FaceclawBleCommunicator? = null
|
||||||
private var whisperModel = WhisperModelFiles.forLanguage(null)
|
|
||||||
|
|
||||||
private val host = object : VoiceCaptureHost {
|
private val host = object : VoiceCaptureHost {
|
||||||
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
||||||
@@ -112,7 +122,7 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
|
|
||||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
||||||
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
||||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
|
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
||||||
@@ -158,11 +168,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
session.setOnboardModelKind(kind)
|
session.setOnboardModelKind(kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
|
|
||||||
fun setOnboardLanguage(language: String?) {
|
|
||||||
whisperModel = WhisperModelFiles.forLanguage(language)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||||
@@ -258,8 +263,8 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
*/
|
*/
|
||||||
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
||||||
val whisper = kind == VoiceModelKind.WHISPER
|
val whisper = kind == VoiceModelKind.WHISPER
|
||||||
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
|
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
|
||||||
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
|
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
|
||||||
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
||||||
for (fileName in fileNames) {
|
for (fileName in fileNames) {
|
||||||
val file = File(modelDir, fileName)
|
val file = File(modelDir, fileName)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { NotificationsListLayer } from "../../ui/notifications";
|
import { NotificationsListLayer } from "../../ui/notifications";
|
||||||
import { NotificationFilterLayer } from "../../ui/notification-filter";
|
import { NotificationFilterLayer } from "../../ui/notification-filter";
|
||||||
import { onAndroidNotificationsChanged } from "../../native/notification-icons";
|
import { ALL_NOTIFICATIONS, dismissAllNotifications, onAndroidNotificationsChanged, readActiveNotifications } from "../../native/notification-icons";
|
||||||
import {
|
import {
|
||||||
createInProcessWindow,
|
createInProcessWindow,
|
||||||
YieldAtRootLayer,
|
YieldAtRootLayer,
|
||||||
@@ -32,6 +32,15 @@ export function createNotificationsAppWindow(options: InProcessAppOptions): InPr
|
|||||||
ctx.stack.pop();
|
ctx.stack.pop();
|
||||||
ctx.stack.push(new NotificationFilterLayer());
|
ctx.stack.push(new NotificationFilterLayer());
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
label: "Dismiss all notifications",
|
||||||
|
description: "Dismiss notifications on the phone and glasses. System-protected notifications may remain.",
|
||||||
|
disabled: () => readActiveNotifications(ALL_NOTIFICATIONS).length === 0,
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
ctx.stack.pop();
|
||||||
|
dismissAllNotifications();
|
||||||
|
created.requestRender();
|
||||||
|
},
|
||||||
}],
|
}],
|
||||||
actions: options.actions,
|
actions: options.actions,
|
||||||
baseLayer: new YieldAtRootLayer(new NotificationsListLayer()),
|
baseLayer: new YieldAtRootLayer(new NotificationsListLayer()),
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
|||||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||||
import {
|
import {
|
||||||
getInstalledEvenHubAppById,
|
getInstalledEvenHubAppById,
|
||||||
@@ -483,7 +483,6 @@ class DashboardController {
|
|||||||
this.applyVerticalPositionIfChanged();
|
this.applyVerticalPositionIfChanged();
|
||||||
this.applyAppLayoutsIfChanged();
|
this.applyAppLayoutsIfChanged();
|
||||||
this.syncAssistantBridgeIfChanged();
|
this.syncAssistantBridgeIfChanged();
|
||||||
this.retargetContinuousCapture();
|
|
||||||
this.syncLockScreenSettingIfChanged();
|
this.syncLockScreenSettingIfChanged();
|
||||||
// A Preview color or scaling change should show on the mirror at once,
|
// A Preview color or scaling change should show on the mirror at once,
|
||||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||||
@@ -2198,16 +2197,16 @@ class DashboardController {
|
|||||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||||
// A window that needs its own provider and language sets them for every
|
// A window that needs its own provider and language sets them for every
|
||||||
// capture while it is open, so push-to-talk still shares its stream.
|
// capture while it is open, so push-to-talk still shares its stream.
|
||||||
// Otherwise use the shared transcription-language preference.
|
// Otherwise only Whisper server is told a language (its own setting).
|
||||||
const choice = this.transcriptionChoice();
|
const choice = this.transcriptionChoice();
|
||||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||||
return {
|
return {
|
||||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||||
usePhoneMic: communicator === null,
|
usePhoneMic: communicator === null,
|
||||||
provider,
|
provider,
|
||||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||||
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
openAiApiKey: openAiApiKeySetting.get(),
|
||||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||||
whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerModel: whisperServerModelSetting.get(),
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
|||||||
import { onAnySettingChanged,
|
import { onAnySettingChanged,
|
||||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||||
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||||
@@ -416,7 +416,6 @@ export class IosPreviewController {
|
|||||||
const phone = readPhoneBatteryState()
|
const phone = readPhoneBatteryState()
|
||||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||||
this.offSettings = onAnySettingChanged(() => {
|
this.offSettings = onAnySettingChanged(() => {
|
||||||
this.syncContinuousCapture()
|
|
||||||
this.syncLockSetting()
|
this.syncLockSetting()
|
||||||
this.pushBrightness()
|
this.pushBrightness()
|
||||||
this.relayout()
|
this.relayout()
|
||||||
@@ -1199,13 +1198,14 @@ export class IosPreviewController {
|
|||||||
/**
|
/**
|
||||||
* The transcription provider setting and the keys the cloud ones need. A
|
* The transcription provider setting and the keys the cloud ones need. A
|
||||||
* window that needs its own provider and language (Translate) sets them for
|
* window that needs its own provider and language (Translate) sets them for
|
||||||
* every capture while it is open; otherwise use the shared language preference.
|
* every capture while it is open; otherwise only Whisper server is told a
|
||||||
|
* language (its own setting).
|
||||||
*/
|
*/
|
||||||
private voiceProvider(): VoiceProviderSelection {
|
private voiceProvider(): VoiceProviderSelection {
|
||||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||||
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-32
@@ -9,7 +9,7 @@ declare const java: any;
|
|||||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||||
* directory each model's `dirName` names.
|
* directory each model's `dirName` names.
|
||||||
*
|
*
|
||||||
* Downloadable models, same shape:
|
* Two models, same shape:
|
||||||
* - "moonshine": the original on-device option (three files). The model is
|
* - "moonshine": the original on-device option (three files). The model is
|
||||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||||
* location that earlier releases copied the bundled files to, so
|
* location that earlier releases copied the bundled files to, so
|
||||||
@@ -29,17 +29,12 @@ declare const java: any;
|
|||||||
* used for Moonshine below. Every hash here was computed locally from a
|
* used for Moonshine below. Every hash here was computed locally from a
|
||||||
* freshly downloaded copy of the file, not copied from a doc page.
|
* freshly downloaded copy of the file, not copied from a doc page.
|
||||||
*
|
*
|
||||||
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
|
|
||||||
* speech and auto-detection; English-only downloads remain untouched.
|
|
||||||
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
|
||||||
* vocabulary hash computed locally. URLs pin the source revision.
|
|
||||||
*
|
|
||||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||||
* here is multiple files rather than one; they download sequentially through
|
* here is multiple files rather than one; they download sequentially through
|
||||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
|
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||||
|
|
||||||
type AsrModelFile = {
|
type AsrModelFile = {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -110,30 +105,6 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
|||||||
],
|
],
|
||||||
totalBytes: 160626066,
|
totalBytes: 160626066,
|
||||||
},
|
},
|
||||||
|
|
||||||
"whisper-base": {
|
|
||||||
label: "Whisper (multilingual, base)",
|
|
||||||
dirName: "sherpa-onnx-whisper-base-int8",
|
|
||||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
|
|
||||||
files: [
|
|
||||||
{
|
|
||||||
name: "base-encoder.int8.onnx",
|
|
||||||
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
|
|
||||||
sizeBytes: 29120534,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "base-decoder.int8.onnx",
|
|
||||||
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
|
|
||||||
sizeBytes: 130672026,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "base-tokens.txt",
|
|
||||||
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
|
||||||
sizeBytes: 816730,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
totalBytes: 160609290,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type AsrModelState = {
|
export type AsrModelState = {
|
||||||
@@ -158,7 +129,6 @@ function freshRuntime(): ModelRuntime {
|
|||||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||||
moonshine: freshRuntime(),
|
moonshine: freshRuntime(),
|
||||||
"whisper-base-en": freshRuntime(),
|
"whisper-base-en": freshRuntime(),
|
||||||
"whisper-base": freshRuntime(),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
function modelDirPath(id: AsrModelId): string {
|
function modelDirPath(id: AsrModelId): string {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||||
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||||
import { SonioxSttClient } from "./soniox-stt";
|
import { SonioxSttClient } from "./soniox-stt";
|
||||||
import type { VoiceProviderKind } from "./voice-control";
|
import type { VoiceProviderKind } from "./voice-control";
|
||||||
@@ -14,12 +14,11 @@ export type VoiceProviderSelection = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
/**
|
/**
|
||||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||||
* detect the language. Also used by on-device Whisper.
|
* detect the language. Cloud providers only.
|
||||||
*/
|
*/
|
||||||
language?: string;
|
language?: string;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
openAiTranscriptionModel?: OpenAiSttModel;
|
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
@@ -53,7 +52,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
|||||||
whisper: {
|
whisper: {
|
||||||
name: "OpenAI",
|
name: "OpenAI",
|
||||||
apiKey: (selection) => selection.openAiApiKey,
|
apiKey: (selection) => selection.openAiApiKey,
|
||||||
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||||
},
|
},
|
||||||
"whisper-server": {
|
"whisper-server": {
|
||||||
name: "Whisper server",
|
name: "Whisper server",
|
||||||
@@ -77,10 +76,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
|||||||
* What a capture transcribes with: the provider and the language it is told
|
* What a capture transcribes with: the provider and the language it is told
|
||||||
* to expect. A running capture is shared only by holders asking for the same.
|
* to expect. A running capture is shared only by holders asking for the same.
|
||||||
*/
|
*/
|
||||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||||
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||||
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
|
||||||
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import type { GrayImage } from '../graphics/image'
|
|||||||
import { pebbleIcon } from '../graphics/pebble-icons'
|
import { pebbleIcon } from '../graphics/pebble-icons'
|
||||||
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
|
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
|
||||||
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
|
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
|
||||||
import { rememberNotificationSources } from './notification-sources'
|
import { rememberNotificationSources, shouldShowNotificationOnGlasses } from './notification-sources'
|
||||||
import type { AndroidNotification } from './notification-types'
|
import type { AndroidNotification } from './notification-types'
|
||||||
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
|
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
|
||||||
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
|
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
|
||||||
@@ -24,8 +24,23 @@ export function onIosNotificationPopup(listener: (key: string) => void): () => v
|
|||||||
export function iosNotificationMessage(): string { return active?.statusMessage ?? ANCS_CONNECT_MESSAGE }
|
export function iosNotificationMessage(): string { return active?.statusMessage ?? ANCS_CONNECT_MESSAGE }
|
||||||
export function iosNotificationState(): AncsState { return active?.state ?? 'disconnected' }
|
export function iosNotificationState(): AncsState { return active?.state ?? 'disconnected' }
|
||||||
export function readActiveNotifications(maxNotifications = 50) { return active?.read(maxNotifications) ?? [] }
|
export function readActiveNotifications(maxNotifications = 50) { return active?.read(maxNotifications) ?? [] }
|
||||||
|
export function readVisibleNotifications(maxNotifications = 50) {
|
||||||
|
return readActiveNotifications(ALL_NOTIFICATIONS)
|
||||||
|
.filter(notification => shouldShowNotificationOnGlasses(notification.packageName))
|
||||||
|
.slice(0, Math.max(0, maxNotifications))
|
||||||
|
}
|
||||||
export function invokeNotificationAction(key: string, index: number): boolean { return active?.action(key,index) ?? false }
|
export function invokeNotificationAction(key: string, index: number): boolean { return active?.action(key,index) ?? false }
|
||||||
export function dismissNotification(key: string): boolean { return active?.dismiss(key) ?? false }
|
export function dismissNotification(key: string): boolean { return active?.dismiss(key) ?? false }
|
||||||
|
export function dismissAllNotifications(): boolean {
|
||||||
|
const client = active
|
||||||
|
if (!client) return false
|
||||||
|
// Snapshot: removals may arrive while dismissal requests are being sent.
|
||||||
|
let accepted = true
|
||||||
|
for (const notification of [...client.read(ALL_NOTIFICATIONS)]) {
|
||||||
|
if (!client.dismiss(notification.key)) accepted = false
|
||||||
|
}
|
||||||
|
return accepted
|
||||||
|
}
|
||||||
export function onAndroidNotificationPosted(listener: (key: string) => void): () => void {
|
export function onAndroidNotificationPosted(listener: (key: string) => void): () => void {
|
||||||
listeners.add(listener); return () => { listeners.delete(listener) }
|
listeners.add(listener); return () => { listeners.delete(listener) }
|
||||||
}
|
}
|
||||||
@@ -44,7 +59,7 @@ function notificationIcon(notification: AndroidNotification): GrayImage {
|
|||||||
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
|
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
|
||||||
// One icon per source, standing for its first notification.
|
// One icon per source, standing for its first notification.
|
||||||
const sources = new Map<string, AndroidNotification>()
|
const sources = new Map<string, AndroidNotification>()
|
||||||
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
|
for (const n of readVisibleNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
|
||||||
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
|
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
|
||||||
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
|
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { GrayImage } from "../graphics/image";
|
import { GrayImage } from "../graphics/image";
|
||||||
import { logCurrent, spanCurrent } from "./frame-timings";
|
import { logCurrent, spanCurrent } from "./frame-timings";
|
||||||
import { toUint8Array } from "../util/array-util";
|
import { toUint8Array } from "../util/array-util";
|
||||||
import { rememberNotificationSources } from "./notification-sources";
|
import { rememberNotificationSources, shouldShowNotificationOnGlasses } from "./notification-sources";
|
||||||
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
|
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
|
||||||
import { NOTIFICATION_ICON_SIZE } from "./notification-types";
|
import { NOTIFICATION_ICON_SIZE } from "./notification-types";
|
||||||
|
|
||||||
@@ -21,6 +21,7 @@ const ICON_CACHE_MS = 60_000;
|
|||||||
|
|
||||||
let cachedIcons: GrayImage[] = [];
|
let cachedIcons: GrayImage[] = [];
|
||||||
let cachedKeys: string[] = [];
|
let cachedKeys: string[] = [];
|
||||||
|
let cachedPackages = new Map<string, string>();
|
||||||
let cachedAtMs = 0;
|
let cachedAtMs = 0;
|
||||||
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
|
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
|
||||||
const KEYED_ICON_CACHE_MAX = 128;
|
const KEYED_ICON_CACHE_MAX = 128;
|
||||||
@@ -95,6 +96,7 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
|
|||||||
keys.push(index < keyList.size() ? String(keyList.get(index)) : "");
|
keys.push(index < keyList.size() ? String(keyList.get(index)) : "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cachedPackages = new Map(readActiveNotifications(ALL_NOTIFICATIONS).map(n => [n.key, n.packageName]));
|
||||||
cachedIcons = icons;
|
cachedIcons = icons;
|
||||||
cachedKeys = keys;
|
cachedKeys = keys;
|
||||||
cachedAtMs = now;
|
cachedAtMs = now;
|
||||||
@@ -103,7 +105,12 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
|
|||||||
|
|
||||||
/** Copies of the first maxIcons cached icons, with their keys. */
|
/** Copies of the first maxIcons cached icons, with their keys. */
|
||||||
function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult {
|
function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult {
|
||||||
return { icons: cachedIcons.slice(0, maxIcons).map(icon => icon.clone()), keys: cachedKeys.slice(0, maxIcons), stale };
|
// Reapply preferences even to a warm/stale cache: toggles take effect immediately.
|
||||||
|
const indices = cachedKeys.map((_, index) => index).filter(index => {
|
||||||
|
const packageName = cachedPackages.get(cachedKeys[index]!);
|
||||||
|
return packageName !== undefined && shouldShowNotificationOnGlasses(packageName);
|
||||||
|
}).slice(0, maxIcons);
|
||||||
|
return { icons: indices.map(index => cachedIcons[index]!.clone()), keys: indices.map(index => cachedKeys[index]!), stale };
|
||||||
}
|
}
|
||||||
|
|
||||||
export type NotificationIconResult = {
|
export type NotificationIconResult = {
|
||||||
@@ -318,6 +325,13 @@ export function readActiveNotifications(maxNotifications = 50): AndroidNotificat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Glasses display only; discover all sources and filter before the display limit. */
|
||||||
|
export function readVisibleNotifications(maxNotifications = 50): AndroidNotification[] {
|
||||||
|
return readActiveNotifications(ALL_NOTIFICATIONS)
|
||||||
|
.filter(notification => shouldShowNotificationOnGlasses(notification.packageName))
|
||||||
|
.slice(0, Math.max(0, maxNotifications));
|
||||||
|
}
|
||||||
|
|
||||||
export function invokeNotificationAction(notificationKey: string, actionIndex: number): boolean {
|
export function invokeNotificationAction(notificationKey: string, actionIndex: number): boolean {
|
||||||
if (!global.isAndroid || !notificationKey) return false;
|
if (!global.isAndroid || !notificationKey) return false;
|
||||||
invalidateIconCaches();
|
invalidateIconCaches();
|
||||||
@@ -337,6 +351,13 @@ export function dismissNotification(notificationKey: string): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Explicit user action: dismiss on the phone as well as on the glasses. */
|
||||||
|
export function dismissAllNotifications(): boolean {
|
||||||
|
if (!global.isAndroid) return false;
|
||||||
|
invalidateIconCaches();
|
||||||
|
return Boolean(com.faceclaw.app.FaceclawMediaNotificationListenerService.dismissAllNotifications());
|
||||||
|
}
|
||||||
|
|
||||||
export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void {
|
export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void {
|
||||||
return addNotificationListener(notificationPostedListeners, listener);
|
return addNotificationListener(notificationPostedListeners, listener);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,16 @@
|
|||||||
import { getStringSetting, setStringSetting } from "./settings-store";
|
import { getStringSetting, setStringSetting } from "./settings-store";
|
||||||
|
|
||||||
const SOURCES_KEY = "notifications.sources";
|
const SOURCES_KEY = "notifications.sources";
|
||||||
|
const DEFAULT_KEY = "notifications.newAppsEnabledByDefault";
|
||||||
|
|
||||||
|
/** Unset preserves the behavior of existing installations. */
|
||||||
|
export function newNotificationAppsEnabledByDefault(): boolean {
|
||||||
|
return getStringSetting(DEFAULT_KEY, "true") !== "false";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setNewNotificationAppsEnabledByDefault(enabled: boolean): void {
|
||||||
|
setStringSetting(DEFAULT_KEY, String(enabled));
|
||||||
|
}
|
||||||
|
|
||||||
export type NotificationSource = {
|
export type NotificationSource = {
|
||||||
packageName: string;
|
packageName: string;
|
||||||
@@ -8,8 +18,15 @@ export type NotificationSource = {
|
|||||||
showOnGlasses: boolean;
|
showOnGlasses: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type SavedSource = Omit<NotificationSource, "showOnGlasses"> & { showOnGlasses?: boolean };
|
||||||
|
|
||||||
/** Sources outlive their active notifications so a muted app can always be re-enabled. */
|
/** Sources outlive their active notifications so a muted app can always be re-enabled. */
|
||||||
export function readNotificationSources(): NotificationSource[] {
|
export function readNotificationSources(): NotificationSource[] {
|
||||||
|
const enabled = newNotificationAppsEnabledByDefault();
|
||||||
|
return readSavedSources().map((source) => ({ ...source, showOnGlasses: source.showOnGlasses ?? enabled }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function readSavedSources(): SavedSource[] {
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]"));
|
const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]"));
|
||||||
if (!Array.isArray(parsed)) return [];
|
if (!Array.isArray(parsed)) return [];
|
||||||
@@ -17,7 +34,7 @@ export function readNotificationSources(): NotificationSource[] {
|
|||||||
.map((source) => ({
|
.map((source) => ({
|
||||||
packageName: source.packageName,
|
packageName: source.packageName,
|
||||||
appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName,
|
appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName,
|
||||||
showOnGlasses: source.showOnGlasses !== false,
|
showOnGlasses: typeof source.showOnGlasses === "boolean" ? source.showOnGlasses : undefined,
|
||||||
}));
|
}));
|
||||||
} catch {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
@@ -25,7 +42,7 @@ export function readNotificationSources(): NotificationSource[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void {
|
export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void {
|
||||||
const sources = new Map(readNotificationSources().map((source) => [source.packageName, source]));
|
const sources = new Map(readSavedSources().map((source) => [source.packageName, source]));
|
||||||
let changed = false;
|
let changed = false;
|
||||||
for (const notification of notifications) {
|
for (const notification of notifications) {
|
||||||
if (!notification.packageName) continue;
|
if (!notification.packageName) continue;
|
||||||
@@ -35,7 +52,7 @@ export function rememberNotificationSources(notifications: { packageName: string
|
|||||||
sources.set(notification.packageName, {
|
sources.set(notification.packageName, {
|
||||||
packageName: notification.packageName,
|
packageName: notification.packageName,
|
||||||
appName,
|
appName,
|
||||||
showOnGlasses: previous?.showOnGlasses ?? true,
|
showOnGlasses: previous?.showOnGlasses,
|
||||||
});
|
});
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
@@ -44,12 +61,13 @@ export function rememberNotificationSources(notifications: { packageName: string
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function shouldShowNotificationOnGlasses(packageName: string): boolean {
|
export function shouldShowNotificationOnGlasses(packageName: string): boolean {
|
||||||
return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses ?? true;
|
return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses
|
||||||
|
?? newNotificationAppsEnabledByDefault();
|
||||||
}
|
}
|
||||||
|
|
||||||
export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void {
|
export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void {
|
||||||
if (!source.packageName) return;
|
if (!source.packageName) return;
|
||||||
const sources = new Map(readNotificationSources().map((item) => [item.packageName, item]));
|
const sources = new Map(readSavedSources().map((item) => [item.packageName, item]));
|
||||||
sources.set(source.packageName, {
|
sources.set(source.packageName, {
|
||||||
packageName: source.packageName,
|
packageName: source.packageName,
|
||||||
appName: source.appName || source.packageName,
|
appName: source.appName || source.packageName,
|
||||||
@@ -57,3 +75,10 @@ export function setNotificationSourceEnabled(source: { packageName: string; appN
|
|||||||
});
|
});
|
||||||
setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
|
setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Set explicit choices only for sources already discovered; leave the future default alone. */
|
||||||
|
export function setAllNotificationSourcesEnabled(enabled: boolean): void {
|
||||||
|
setStringSetting(SOURCES_KEY, JSON.stringify(readSavedSources().map((source) => ({
|
||||||
|
...source, showOnGlasses: enabled,
|
||||||
|
}))));
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||||
* on the Realtime API, with a configurable transcription model).
|
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
||||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||||
*
|
*
|
||||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||||
@@ -16,11 +16,10 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||||
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
const MODEL_ID = "gpt-realtime-whisper";
|
||||||
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
|
||||||
const TARGET_SAMPLE_RATE = 24000;
|
const TARGET_SAMPLE_RATE = 24000;
|
||||||
|
|
||||||
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
export type OpenAiSttOptions = CloudSttOptions;
|
||||||
|
|
||||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||||
private ws: SocketConnection | null = null;
|
private ws: SocketConnection | null = null;
|
||||||
@@ -41,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
onOpen: () => {
|
onOpen: () => {
|
||||||
if (this.closed) return;
|
if (this.closed) return;
|
||||||
this.open = true;
|
this.open = true;
|
||||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||||
for (const chunk of this.pendingChunks.splice(0)) {
|
for (const chunk of this.pendingChunks.splice(0)) {
|
||||||
this.sendChunk(chunk);
|
this.sendChunk(chunk);
|
||||||
}
|
}
|
||||||
@@ -166,9 +165,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.options.onError(message.error?.code === "model_not_found"
|
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||||
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
|
||||||
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
|
||||||
return;
|
return;
|
||||||
default:
|
default:
|
||||||
return;
|
return;
|
||||||
@@ -204,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
function sessionConfig(language: string): object {
|
||||||
return {
|
return {
|
||||||
type: "session.update",
|
type: "session.update",
|
||||||
session: {
|
session: {
|
||||||
@@ -212,7 +209,7 @@ function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_
|
|||||||
audio: {
|
audio: {
|
||||||
input: {
|
input: {
|
||||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||||
transcription: { model, ...(language ? { language } : {}) },
|
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||||
turn_detection: null,
|
turn_detection: null,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
/** Languages supported by both Whisper backends, in picker order. */
|
|
||||||
export const TRANSCRIPTION_LANGUAGES = {
|
|
||||||
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
|
|
||||||
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
|
|
||||||
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
|
|
||||||
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
|
|
||||||
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
|
|
||||||
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
|
|
||||||
|
|
||||||
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
|
|
||||||
export function resolveTranscriptionLanguage(
|
|
||||||
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
|
||||||
): string {
|
|
||||||
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
|
||||||
if (provider === "onboard") return "";
|
|
||||||
if (choice === "default") {
|
|
||||||
if (provider === "onboard-whisper") return "en";
|
|
||||||
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
|
|
||||||
}
|
|
||||||
return choice === "auto" ? "" : choice;
|
|
||||||
}
|
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { OpenAiSttModel } from "./openai-stt";
|
|
||||||
import { Utils } from "@nativescript/core";
|
import { Utils } from "@nativescript/core";
|
||||||
|
|
||||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||||
@@ -25,7 +24,7 @@ export type VoiceControlState = {
|
|||||||
detail: string;
|
detail: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||||
@@ -55,12 +54,11 @@ export type PushToTalkOptions = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
openAiTranscriptionModel?: OpenAiSttModel;
|
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
whisperServerModel?: string;
|
whisperServerModel?: string;
|
||||||
whisperServerApiKey?: string;
|
whisperServerApiKey?: string;
|
||||||
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
|
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||||
language?: string;
|
language?: string;
|
||||||
saveRecording: boolean;
|
saveRecording: boolean;
|
||||||
/**
|
/**
|
||||||
@@ -398,7 +396,6 @@ export class FaceclawVoiceControlBridge {
|
|||||||
// Which on-device model to load; a no-op setter for every provider except
|
// Which on-device model to load; a no-op setter for every provider except
|
||||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||||
this.controller?.setOnboardLanguage(options.language ?? "en");
|
|
||||||
const captureId = this.nextNativeCaptureId++;
|
const captureId = this.nextNativeCaptureId++;
|
||||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||||
this.controller?.start("onboard", captureId);
|
this.controller?.start("onboard", captureId);
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { OpenAiSttModel } from "../native/openai-stt";
|
|
||||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||||
@@ -19,7 +18,6 @@ import {
|
|||||||
} from "~/assistant/models";
|
} from "~/assistant/models";
|
||||||
import { isLocalModelReady } from "../native/local-llm";
|
import { isLocalModelReady } from "../native/local-llm";
|
||||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||||
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
|
|
||||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||||
@@ -1138,35 +1136,9 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
|||||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
|
||||||
id: "openai-transcription-model",
|
|
||||||
label: "OpenAI transcription model",
|
|
||||||
storageKey: "voice.openAiTranscriptionModel",
|
|
||||||
defaultValue: "gpt-realtime-whisper",
|
|
||||||
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
|
||||||
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
|
||||||
});
|
|
||||||
|
|
||||||
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
|
||||||
id: "voice-language",
|
|
||||||
label: "Transcription language",
|
|
||||||
storageKey: "voice.language",
|
|
||||||
defaultValue: "default",
|
|
||||||
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
|
|
||||||
formatValue: (value) => value === "default" ? "Provider default"
|
|
||||||
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
|
||||||
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
|
||||||
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
|
||||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
|
|
||||||
});
|
|
||||||
|
|
||||||
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
|
||||||
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
|
|
||||||
}
|
|
||||||
|
|
||||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||||
id: "whisper-server-language",
|
id: "whisper-server-language",
|
||||||
label: "Whisper server language",
|
label: "Whisper language",
|
||||||
storageKey: "voice.whisperServerLanguage",
|
storageKey: "voice.whisperServerLanguage",
|
||||||
defaultValue: "",
|
defaultValue: "",
|
||||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||||
@@ -1174,7 +1146,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
|
|||||||
formatValue: (value) => value || "Auto-detect",
|
formatValue: (value) => value || "Auto-detect",
|
||||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||||
description:
|
description:
|
||||||
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
|
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ import {
|
|||||||
mapboxApiKeySetting,
|
mapboxApiKeySetting,
|
||||||
mirrorTouchSetting,
|
mirrorTouchSetting,
|
||||||
openAiApiKeySetting,
|
openAiApiKeySetting,
|
||||||
openAiTranscriptionModelSetting,
|
|
||||||
previewColorSetting,
|
previewColorSetting,
|
||||||
previewScalingSetting,
|
previewScalingSetting,
|
||||||
phoneRotationSetting,
|
phoneRotationSetting,
|
||||||
@@ -88,7 +87,6 @@ import {
|
|||||||
toggleSettingMenuItem,
|
toggleSettingMenuItem,
|
||||||
useMicControlSetting,
|
useMicControlSetting,
|
||||||
voiceProviderSetting,
|
voiceProviderSetting,
|
||||||
voiceLanguageSetting,
|
|
||||||
screenTimeoutSetting,
|
screenTimeoutSetting,
|
||||||
wakeToLastAppSetting,
|
wakeToLastAppSetting,
|
||||||
wakeWordActionSetting,
|
wakeWordActionSetting,
|
||||||
@@ -145,11 +143,9 @@ function settingsSections(): SettingsSection[] {
|
|||||||
items: [
|
items: [
|
||||||
enumSettingMenuItem(wakeWordActionSetting),
|
enumSettingMenuItem(wakeWordActionSetting),
|
||||||
enumSettingMenuItem(voiceProviderSetting),
|
enumSettingMenuItem(voiceProviderSetting),
|
||||||
enumSettingMenuItem(voiceLanguageSetting),
|
|
||||||
...whisperServerMenuItems(),
|
...whisperServerMenuItems(),
|
||||||
asrModelMenuItem("moonshine"),
|
asrModelMenuItem("moonshine"),
|
||||||
asrModelMenuItem("whisper-base-en"),
|
asrModelMenuItem("whisper-base-en"),
|
||||||
asrModelMenuItem("whisper-base"),
|
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -288,7 +284,7 @@ function settingsSections(): SettingsSection[] {
|
|||||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||||
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||||
item.label !== screenTimeoutSetting.label) };
|
item.label !== screenTimeoutSetting.label) };
|
||||||
return section;
|
return section;
|
||||||
@@ -531,7 +527,6 @@ function asrModelStatusText(id: AsrModelId): string {
|
|||||||
*/
|
*/
|
||||||
function whisperServerMenuItems(): MenuItem[] {
|
function whisperServerMenuItems(): MenuItem[] {
|
||||||
return [
|
return [
|
||||||
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
|
||||||
textSettingMenuItem(whisperServerUrlSetting),
|
textSettingMenuItem(whisperServerUrlSetting),
|
||||||
textSettingMenuItem(whisperServerModelSetting),
|
textSettingMenuItem(whisperServerModelSetting),
|
||||||
textSettingMenuItem(whisperServerLanguageSetting),
|
textSettingMenuItem(whisperServerLanguageSetting),
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { getDefaultSmallFont } from "../graphics/ui-fonts";
|
|||||||
import { GrayImage } from "../graphics/image";
|
import { GrayImage } from "../graphics/image";
|
||||||
import { truncateText } from "../graphics/textwrap";
|
import { truncateText } from "../graphics/textwrap";
|
||||||
import { ALL_NOTIFICATIONS, readActiveNotifications } from "../native/notification-icons";
|
import { ALL_NOTIFICATIONS, readActiveNotifications } from "../native/notification-icons";
|
||||||
import { readNotificationSources, setNotificationSourceEnabled, shouldShowNotificationOnGlasses } from "../native/notification-sources";
|
import { newNotificationAppsEnabledByDefault, setNewNotificationAppsEnabledByDefault, setAllNotificationSourcesEnabled, readNotificationSources, setNotificationSourceEnabled, shouldShowNotificationOnGlasses } from "../native/notification-sources";
|
||||||
import { type InputEvent } from "./gestures";
|
import { type InputEvent } from "./gestures";
|
||||||
import { type Layer, type LayerContext, type PaintBelow } from "./layers";
|
import { type Layer, type LayerContext, type PaintBelow } from "./layers";
|
||||||
import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu";
|
import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu";
|
||||||
@@ -11,14 +11,26 @@ import { drawToggleMenuItem, MenuLayer, type MenuItem } from "./menu";
|
|||||||
export class NotificationFilterLayer implements Layer {
|
export class NotificationFilterLayer implements Layer {
|
||||||
private menu: MenuLayer | null = null;
|
private menu: MenuLayer | null = null;
|
||||||
private menuWidth = 0;
|
private menuWidth = 0;
|
||||||
/** The package behind each current row, parallel to the menu's items. */
|
/** Stable row keys, including controls, parallel to the menu's items. */
|
||||||
private packages: string[] = [];
|
private packages: string[] = [];
|
||||||
|
|
||||||
private currentMenu(ctx: LayerContext): MenuLayer {
|
private currentMenu(ctx: LayerContext): MenuLayer {
|
||||||
readActiveNotifications(ALL_NOTIFICATIONS);
|
readActiveNotifications(ALL_NOTIFICATIONS);
|
||||||
const sources = readNotificationSources().sort((a, b) =>
|
const sources = readNotificationSources().sort((a, b) =>
|
||||||
a.appName.localeCompare(b.appName) || a.packageName.localeCompare(b.packageName));
|
a.appName.localeCompare(b.appName) || a.packageName.localeCompare(b.packageName));
|
||||||
const items: MenuItem[] = sources.map((source) => ({
|
const items: MenuItem[] = [{
|
||||||
|
label: "New apps enabled by default",
|
||||||
|
onSelect: () => setNewNotificationAppsEnabledByDefault(!newNotificationAppsEnabledByDefault()),
|
||||||
|
render: ({ image, x, y, width, selected }) => {
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
drawToggleMenuItem(image, font, x, y, width,
|
||||||
|
truncateText(font, "New apps enabled by default", width - 48), newNotificationAppsEnabledByDefault(), selected);
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
label: "Enable all", onSelect: () => setAllNotificationSourcesEnabled(true),
|
||||||
|
}, {
|
||||||
|
label: "Disable all", onSelect: () => setAllNotificationSourcesEnabled(false),
|
||||||
|
}, ...sources.map<MenuItem>((source) => ({
|
||||||
label: source.appName,
|
label: source.appName,
|
||||||
onSelect: () => setNotificationSourceEnabled(source, !shouldShowNotificationOnGlasses(source.packageName)),
|
onSelect: () => setNotificationSourceEnabled(source, !shouldShowNotificationOnGlasses(source.packageName)),
|
||||||
render: ({ image, x, y, width, selected }) => {
|
render: ({ image, x, y, width, selected }) => {
|
||||||
@@ -26,11 +38,11 @@ export class NotificationFilterLayer implements Layer {
|
|||||||
drawToggleMenuItem(image, font, x, y, width, truncateText(font, source.appName, width - 48),
|
drawToggleMenuItem(image, font, x, y, width, truncateText(font, source.appName, width - 48),
|
||||||
shouldShowNotificationOnGlasses(source.packageName), selected);
|
shouldShowNotificationOnGlasses(source.packageName), selected);
|
||||||
},
|
},
|
||||||
}));
|
}))];
|
||||||
if (!items.length) items.push({ label: "No notification sources discovered yet.", disabled: true, onSelect: () => {} });
|
if (!sources.length) items.push({ label: "No notification sources discovered yet.", disabled: true, onSelect: () => {} });
|
||||||
const selectedPackage = this.menu?.selectedIndex != null ? this.packages[this.menu.selectedIndex] : undefined;
|
const selectedPackage = this.menu?.selectedIndex != null ? this.packages[this.menu.selectedIndex] : undefined;
|
||||||
const index = sources.findIndex((source) => source.packageName === selectedPackage);
|
this.packages = ["default", "enable-all", "disable-all", ...sources.map((source) => `source:${source.packageName}`)];
|
||||||
this.packages = sources.map((source) => source.packageName);
|
const index = selectedPackage === undefined ? 0 : this.packages.indexOf(selectedPackage);
|
||||||
|
|
||||||
const width = ctx.stack.getBaseSize().width;
|
const width = ctx.stack.getBaseSize().width;
|
||||||
if (!this.menu || width !== this.menuWidth) {
|
if (!this.menu || width !== this.menuWidth) {
|
||||||
@@ -38,7 +50,7 @@ export class NotificationFilterLayer implements Layer {
|
|||||||
this.menu = new MenuLayer("Notification filter", items, {
|
this.menu = new MenuLayer("Notification filter", items, {
|
||||||
x: 8, y: 8, width: width - 16,
|
x: 8, y: 8, width: width - 16,
|
||||||
opaque: true, showBorder: false,
|
opaque: true, showBorder: false,
|
||||||
footer: "On = show popups on glasses",
|
footer: "On = show notifications on glasses",
|
||||||
}).selectItem(Math.max(0, index));
|
}).selectItem(Math.max(0, index));
|
||||||
} else {
|
} else {
|
||||||
this.menu.setItems(items, index >= 0 ? index : undefined);
|
this.menu.setItems(items, index >= 0 ? index : undefined);
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import {
|
|||||||
ALL_NOTIFICATIONS,
|
ALL_NOTIFICATIONS,
|
||||||
dismissNotification,
|
dismissNotification,
|
||||||
invokeNotificationAction,
|
invokeNotificationAction,
|
||||||
readActiveNotifications,
|
readVisibleNotifications,
|
||||||
readNotificationCustomView,
|
readNotificationCustomView,
|
||||||
readNotificationIconByKey,
|
readNotificationIconByKey,
|
||||||
readNotificationLargeIcon,
|
readNotificationLargeIcon,
|
||||||
@@ -141,7 +141,7 @@ export class NotificationsListLayer implements Layer {
|
|||||||
const image = new GrayImage(width, height, 0);
|
const image = new GrayImage(width, height, 0);
|
||||||
const cardWidth = width - 2 * CARD_X;
|
const cardWidth = width - 2 * CARD_X;
|
||||||
const cardTextWidth = cardWidth - CARD_TEXT_X - 14;
|
const cardTextWidth = cardWidth - CARD_TEXT_X - 14;
|
||||||
const notifications = readActiveNotifications(MAX_NOTIFICATIONS);
|
const notifications = readVisibleNotifications(MAX_NOTIFICATIONS);
|
||||||
const selectedIndex = this.resolveSelectedIndex(notifications);
|
const selectedIndex = this.resolveSelectedIndex(notifications);
|
||||||
const layouts = notifications.map((notification, index) =>
|
const layouts = notifications.map((notification, index) =>
|
||||||
buildNotificationCardLayout(font, notification, index === selectedIndex, cardTextWidth),
|
buildNotificationCardLayout(font, notification, index === selectedIndex, cardTextWidth),
|
||||||
@@ -185,7 +185,7 @@ export class NotificationsListLayer implements Layer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||||
const notifications = readActiveNotifications(MAX_NOTIFICATIONS);
|
const notifications = readVisibleNotifications(MAX_NOTIFICATIONS);
|
||||||
const selectedIndex = this.resolveSelectedIndex(notifications);
|
const selectedIndex = this.resolveSelectedIndex(notifications);
|
||||||
if (event.type === "double-click") {
|
if (event.type === "double-click") {
|
||||||
// At the app's root this is intercepted by the yield wrapper; reached
|
// At the app's root this is intercepted by the yield wrapper; reached
|
||||||
@@ -252,7 +252,7 @@ export class SingleNotificationLayer implements Layer {
|
|||||||
const font = getDefaultSmallFont();
|
const font = getDefaultSmallFont();
|
||||||
const { width, height } = ctx.stack.getBaseSize();
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
const image = new GrayImage(width, height, 0);
|
const image = new GrayImage(width, height, 0);
|
||||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
|
const notification = readVisibleNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
|
||||||
|
|
||||||
if (!notification) {
|
if (!notification) {
|
||||||
return this.closeUnavailableNotification(ctx, paintBelow);
|
return this.closeUnavailableNotification(ctx, paintBelow);
|
||||||
@@ -273,7 +273,7 @@ export class SingleNotificationLayer implements Layer {
|
|||||||
else await menu.handleInput(event);
|
else await menu.handleInput(event);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
|
const notification = readVisibleNotifications(ALL_NOTIFICATIONS).find((item) => item.key === this.notificationKey);
|
||||||
if (!notification) {
|
if (!notification) {
|
||||||
this.closeUnavailableNotification(ctx);
|
this.closeUnavailableNotification(ctx);
|
||||||
return;
|
return;
|
||||||
@@ -297,7 +297,7 @@ export class SingleNotificationLayer implements Layer {
|
|||||||
this.close(ctx);
|
this.close(ctx);
|
||||||
} else if (item.kind === "action") {
|
} else if (item.kind === "action") {
|
||||||
invokeNotificationAction(this.notificationKey, item.action.index);
|
invokeNotificationAction(this.notificationKey, item.action.index);
|
||||||
if (!readActiveNotifications(ALL_NOTIFICATIONS).some((active) => active.key === this.notificationKey)) {
|
if (!readVisibleNotifications(ALL_NOTIFICATIONS).some((active) => active.key === this.notificationKey)) {
|
||||||
this.closeUnavailableNotification(ctx);
|
this.closeUnavailableNotification(ctx);
|
||||||
}
|
}
|
||||||
} else if (item.kind === "dismiss") {
|
} else if (item.kind === "dismiss") {
|
||||||
@@ -310,7 +310,7 @@ export class SingleNotificationLayer implements Layer {
|
|||||||
menu: new Menu<ConfirmationItem>({
|
menu: new Menu<ConfirmationItem>({
|
||||||
items: [
|
items: [
|
||||||
{ label: "Cancel", run: (runCtx) => this.close(runCtx) },
|
{ label: "Cancel", run: (runCtx) => this.close(runCtx) },
|
||||||
{ label: "Turn off popups", run: (runCtx) => {
|
{ label: "Hide on glasses", run: (runCtx) => {
|
||||||
setNotificationSourceEnabled(notification, false);
|
setNotificationSourceEnabled(notification, false);
|
||||||
this.close(runCtx);
|
this.close(runCtx);
|
||||||
} },
|
} },
|
||||||
@@ -341,7 +341,7 @@ export class SingleNotificationLayer implements Layer {
|
|||||||
};
|
};
|
||||||
menu.paint(image, menuBox, ctx.stack.isFocused());
|
menu.paint(image, menuBox, ctx.stack.isFocused());
|
||||||
const top = menuBox.y + menuBox.height + CONFIRM_PROMPT_GAP;
|
const top = menuBox.y + menuBox.height + CONFIRM_PROMPT_GAP;
|
||||||
const prompt = `Turn off notification popups from ${source.appName || source.packageName}? You can turn them back on in the Notifications app's Notification filter.`;
|
const prompt = `Hide notifications on glasses from ${source.appName || source.packageName}? You can turn them back on in the Notifications app's Notification filter.`;
|
||||||
const lines = wrapText(font, prompt, width - 2 * CONFIRM_X);
|
const lines = wrapText(font, prompt, width - 2 * CONFIRM_X);
|
||||||
const maxLines = Math.max(1, Math.floor((height - top - 12) / lineStep(font)));
|
const maxLines = Math.max(1, Math.floor((height - top - 12) / lineStep(font)));
|
||||||
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
|
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
# Dismiss all notifications
|
||||||
|
|
||||||
|
The Notifications app menu offers Dismiss all notifications, disabled when the
|
||||||
|
list is empty. This explicit action dismisses notifications on the phone too;
|
||||||
|
popup filtering and source/default preferences remain independent.
|
||||||
|
|
||||||
|
Android uses NotificationListenerService.cancelAllNotifications, so dismissal
|
||||||
|
is not limited to the 50 items displayed by the list and Android controls which
|
||||||
|
notifications are removable. The normal removal callbacks update the list and
|
||||||
|
popups. Missing listener access returns false without changing preferences.
|
||||||
|
iOS uses the existing ANCS dismissal operation on a snapshot of all known keys;
|
||||||
|
notifications without a negative action may remain.
|
||||||
|
|
||||||
|
Software tests cover menu action/disabled state, more than 50 notifications,
|
||||||
|
native bridge dispatch, disconnected access and ANCS bulk dismissal.
|
||||||
|
On a device, verify the permission-granted/revoked cases, protected or ongoing
|
||||||
|
notifications, notification groups, and consistency with the phone's tray.
|
||||||
|
Hardware validation remains pending.
|
||||||
|
|
||||||
|
Reference: https://developer.android.com/reference/android/service/notification/NotificationListenerService#cancelAllNotifications()
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# Notification filter validation
|
||||||
|
|
||||||
|
Software: `npm test` and `bash lint.sh app/native/notification-sources.ts app/ui/notification-filter.ts`.
|
||||||
|
The notification suite covers legacy preferences, reloads of the settings store,
|
||||||
|
discovery, default changes without overwriting exceptions, both bulk actions,
|
||||||
|
empty lists, stable menu selection and popup suppression before wake.
|
||||||
|
|
||||||
|
Hardware validation (pending; no glasses tested):
|
||||||
|
|
||||||
|
1. On an existing installation, check that enabled and muted applications retain
|
||||||
|
their behavior before touching the new default.
|
||||||
|
2. Turn “New apps enabled by default” off; restart the app. Send a notification
|
||||||
|
from a previously unknown package: it must appear in the filter and on the
|
||||||
|
phone, without waking the glasses or showing a glasses popup.
|
||||||
|
3. Enable that package individually; send another notification and check its popup.
|
||||||
|
4. Change the default in both directions; verify individual exceptions survive.
|
||||||
|
5. Use “Disable all”, then discover another package: it follows the default.
|
||||||
|
Repeat with “Enable all” while the default is off.
|
||||||
|
6. Restart and repeat notifications from enabled and disabled sources. Check that
|
||||||
|
bulk actions and filtering never dismiss notifications on the phone.
|
||||||
|
|
||||||
|
The same source preferences now filter glasses popups, the Notifications list,
|
||||||
|
and the notification tray/status icons. Filtering happens before display limits;
|
||||||
|
source discovery and the phone notification lifecycle remain unfiltered.
|
||||||
|
Cached tray icons reapply the current preferences on each read, including stale
|
||||||
|
reads, so a changed toggle cannot leave a muted icon visible.
|
||||||
|
|
||||||
|
Additional software coverage: Android list limits and warm/stale icon caches,
|
||||||
|
Disable all / re-enable without a new post, and iOS list/tray filtering.
|
||||||
|
|
||||||
|
Additional hardware validation (pending): with mixed enabled/disabled sources,
|
||||||
|
check the list and status bar together. Toggle a source while its notification
|
||||||
|
is already active, then Disable all and Enable all. Verify immediate removal /
|
||||||
|
restoration without a new notification, including after restart, and verify
|
||||||
|
notifications remain available in the phone notification shade.
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
# Whisper transcription languages
|
|
||||||
|
|
||||||
Settings > Voice > Transcription language selects French, English, 27 other
|
|
||||||
languages, or auto-detection. It applies to ordinary dictation and continuous
|
|
||||||
transcription using OpenAI, on-device Whisper, and the other cloud providers.
|
|
||||||
Translate retains its own spoken-language preference. Moonshine remains
|
|
||||||
English-only; Apple's on-device speech recognizer retains its own locale.
|
|
||||||
|
|
||||||
The initial **Provider default** preserves existing installations: English for
|
|
||||||
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
|
|
||||||
Whisper server language for self-hosted Whisper. Selecting another shared
|
|
||||||
language overrides the server language without deleting it. Returning to Provider
|
|
||||||
default restores it. Settings persist through the existing settings store.
|
|
||||||
|
|
||||||
For French or auto-detection with **On-device (Whisper)**, download
|
|
||||||
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
|
|
||||||
English uses the existing English-only base model; neither model is deleted or
|
|
||||||
downloaded automatically. A missing model uses the existing “Voice model not
|
|
||||||
downloaded” status. Language changes restart active continuous transcription;
|
|
||||||
other captures use the new language when started.
|
|
||||||
|
|
||||||
## Software validation
|
|
||||||
|
|
||||||
- JavaScript regression tests exercise stored settings/reload, compatible
|
|
||||||
defaults, French and auto-detection, Translate overrides in Android/iOS capture
|
|
||||||
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
|
|
||||||
- Kotlin tests exercise English/French/other/automatic model selection and file
|
|
||||||
names. Android release compilation checks the actual sherpa-onnx bridge.
|
|
||||||
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
|
|
||||||
the existing resumable downloader. Encoder/decoder hashes and sizes come from
|
|
||||||
repository LFS metadata; the vocabulary hash was computed from its contents.
|
|
||||||
|
|
||||||
## Device validation still required
|
|
||||||
|
|
||||||
With both phone microphone preview and glasses microphone, check French dictation
|
|
||||||
(including accents), auto-detection, English rollback, continuous capture while
|
|
||||||
changing language, cancellation/resumption of model download, and persistence
|
|
||||||
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
|
|
||||||
No paid API call, microphone recording, or firmware update is needed for the
|
|
||||||
software checks above; recognition accuracy and latency require device testing.
|
|
||||||
|
|
||||||
## References
|
|
||||||
|
|
||||||
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
|
||||||
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
|
||||||
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
|
||||||
|
|
||||||
## OpenAI transcription model selection
|
|
||||||
|
|
||||||
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
|
||||||
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
|
||||||
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
|
||||||
The unset default retains gpt-realtime-whisper for existing installations; no
|
|
||||||
provider, language or Assistant model preference is overwritten. This setting
|
|
||||||
applies to Translate's speech recognition as well as Transcribe and dictation.
|
|
||||||
The two transcribe alternatives produce transcript deltas after a committed
|
|
||||||
audio segment; existing pause segmentation and push-to-talk release supply the
|
|
||||||
commits. Choosing a model retargets active continuous capture.
|
|
||||||
|
|
||||||
Model access errors name the requested model before the long project error and
|
|
||||||
point to the setting. Factory/session-wire tests cover model, language, manual
|
|
||||||
commit and final transcript behavior; settings tests cover persistence and the
|
|
||||||
Android/iOS option plumbing. No real API inference was performed; actual project
|
|
||||||
access, recognition and latency remain to be checked on a device. The model list
|
|
||||||
provided by the operator establishes account visibility, not a successful audio
|
|
||||||
request.
|
|
||||||
|
|
||||||
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
|
||||||
-18
@@ -1,18 +0,0 @@
|
|||||||
package com.faceclaw.app
|
|
||||||
|
|
||||||
/** The English-only files remain usable after upgrading; other languages need multilingual base. */
|
|
||||||
data class WhisperModelFiles(val language: String) {
|
|
||||||
val englishOnly: Boolean get() = language == "en"
|
|
||||||
val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
|
|
||||||
private val prefix: String get() = if (englishOnly) "base.en" else "base"
|
|
||||||
val encoder: String get() = "$prefix-encoder.int8.onnx"
|
|
||||||
val decoder: String get() = "$prefix-decoder.int8.onnx"
|
|
||||||
val tokens: String get() = "$prefix-tokens.txt"
|
|
||||||
val files: List<String> get() = listOf(encoder, decoder, tokens)
|
|
||||||
|
|
||||||
companion object {
|
|
||||||
/** Null preserves the old English default; an empty code enables automatic detection. */
|
|
||||||
fun forLanguage(language: String?): WhisperModelFiles =
|
|
||||||
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -55,7 +55,6 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
|
|||||||
ASR_MODELS: {
|
ASR_MODELS: {
|
||||||
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
||||||
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
||||||
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
|
|
||||||
},
|
},
|
||||||
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
||||||
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ function settingsFixture(store = new Map()) {
|
|||||||
};
|
};
|
||||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
|
||||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||||
}};
|
}};
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||||
|
|||||||
@@ -207,8 +207,6 @@ test('provider selection falls back to on-device without a key and connects the
|
|||||||
// A running capture is shared only on the same provider and language.
|
// A running capture is shared only on the same provider and language.
|
||||||
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
||||||
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
||||||
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
|
|
||||||
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
|
|
||||||
const statuses = [];
|
const statuses = [];
|
||||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||||
@@ -227,13 +225,13 @@ test('provider selection falls back to on-device without a key and connects the
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
||||||
for (const language of ['fr', 'ja', '']) {
|
for (const language of ['ja', '']) {
|
||||||
const elevenlabs = harness('elevenlabs', true, { language });
|
const elevenlabs = harness('elevenlabs', true, { language });
|
||||||
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
||||||
|
|
||||||
const soniox = harness('soniox', true, { language });
|
const soniox = harness('soniox', true, { language });
|
||||||
soniox.open();
|
soniox.open();
|
||||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
|
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
|
||||||
|
|
||||||
const openai = harness('openai', true, { language });
|
const openai = harness('openai', true, { language });
|
||||||
openai.open();
|
openai.open();
|
||||||
@@ -242,43 +240,3 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
|||||||
assert.equal(transcription.language, language || undefined);
|
assert.equal(transcription.language, language || undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
|
||||||
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
|
||||||
const h = harness('openai', true, { model, language: 'fr' });
|
|
||||||
h.open();
|
|
||||||
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
|
||||||
assert.equal(input.transcription.model, model);
|
|
||||||
assert.equal(input.transcription.language, 'fr');
|
|
||||||
assert.equal(input.turn_detection, null);
|
|
||||||
h.client.acceptPcm(pcm(1200, 150));
|
|
||||||
h.client.acceptPcm(pcm(0, 1500));
|
|
||||||
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
|
||||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
|
||||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
|
||||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
|
||||||
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
|
||||||
assert.equal(h.events.at(-1).isFinal, true);
|
|
||||||
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
|
||||||
assert.ok(h.errors[0].includes(model));
|
|
||||||
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
|
||||||
const created = [];
|
|
||||||
const load = loader({ setTimeout, clearTimeout }, {
|
|
||||||
'./openai-stt': {
|
|
||||||
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
|
||||||
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
|
||||||
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
|
||||||
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
|
||||||
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
|
||||||
assert.equal(created[0].language, 'fr');
|
|
||||||
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
|
||||||
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
|
||||||
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
|
||||||
});
|
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { loader } = require('./helpers/load-typescript.cjs');
|
||||||
|
|
||||||
|
test('Notifications menu dismisses all, including beyond the visible list, and leaves filters untouched', () => {
|
||||||
|
let options, active = Array.from({ length: 80 }, (_, i) => ({ key: String(i) }));
|
||||||
|
let dismissals = 0, renders = 0, pops = 0;
|
||||||
|
const load = loader({}, {
|
||||||
|
'../../ui/notifications': { NotificationsListLayer: class {} },
|
||||||
|
'../../ui/notification-filter': { NotificationFilterLayer: class {} },
|
||||||
|
'../../native/notification-icons': {
|
||||||
|
ALL_NOTIFICATIONS: 0x7fffffff,
|
||||||
|
readActiveNotifications(max) { assert.equal(max, 0x7fffffff); return active; },
|
||||||
|
dismissAllNotifications() { dismissals++; active = []; return true; },
|
||||||
|
onAndroidNotificationsChanged: () => () => {},
|
||||||
|
},
|
||||||
|
'../../ui/shell/in-process-window': {
|
||||||
|
YieldAtRootLayer: class {},
|
||||||
|
createInProcessWindow(value) { options = value; return { requestRender() { renders++; } }; },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
load('app/apps/notifications/notifications-app.ts').createNotificationsAppWindow({});
|
||||||
|
const items = options.menuItems();
|
||||||
|
assert.equal(items[0].label, 'Notification filter');
|
||||||
|
const action = items.find(x => x.label === 'Dismiss all notifications');
|
||||||
|
assert.equal(action.disabled(), false);
|
||||||
|
assert.equal(dismissals, 0);
|
||||||
|
action.onSelect({ stack: { pop() { pops++; } } });
|
||||||
|
assert.equal(dismissals, 1);
|
||||||
|
assert.equal(pops, 1);
|
||||||
|
assert.equal(renders, 1);
|
||||||
|
assert.equal(action.disabled(), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Android bulk action uses the native listener and propagates unavailable access', () => {
|
||||||
|
let calls = 0, allowed = false;
|
||||||
|
const load = loader({ global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||||
|
FaceclawMediaNotificationListenerService: { dismissAllNotifications() { calls++; return allowed; } },
|
||||||
|
} } } }, {
|
||||||
|
'../graphics/image': {}, './frame-timings': {}, '../util/array-util': {},
|
||||||
|
'./notification-sources': {}, './image-files': { PHOTO_TONE: { gamma: 1 } }, './notification-types': { NOTIFICATION_ICON_SIZE: 24 },
|
||||||
|
});
|
||||||
|
const api = load('app/native/notification-icons.ts');
|
||||||
|
assert.equal(api.dismissAllNotifications(), false);
|
||||||
|
allowed = true;
|
||||||
|
assert.equal(api.dismissAllNotifications(), true);
|
||||||
|
assert.equal(calls, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS snapshots all ANCS keys and continues when a notification cannot be dismissed', () => {
|
||||||
|
const load = loader({}, {
|
||||||
|
'../graphics/pebble-icons': {}, '../g2/ancs-client': {}, './notification-app-icons': {}, './notification-sources': {},
|
||||||
|
});
|
||||||
|
const api = load('app/native/notification-icons.ios.ts');
|
||||||
|
assert.equal(api.dismissAllNotifications(), false);
|
||||||
|
const active = Array.from({ length: 80 }, (_, i) => ({ key: String(i) })), attempted = [];
|
||||||
|
api.bindIosNotifications({
|
||||||
|
read(max) { assert.equal(max, api.ALL_NOTIFICATIONS); return active; },
|
||||||
|
dismiss(key) {
|
||||||
|
attempted.push(key);
|
||||||
|
if (key === '1') return false;
|
||||||
|
active.splice(active.findIndex(x => x.key === key), 1);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(api.dismissAllNotifications(), false);
|
||||||
|
assert.equal(attempted.length, 80);
|
||||||
|
assert.deepEqual(active, [{ key: '1' }]);
|
||||||
|
});
|
||||||
@@ -12,7 +12,6 @@ function settingsFixture(store = new Map()) {
|
|||||||
};
|
};
|
||||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
|
||||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||||
}};
|
}};
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||||
|
|||||||
@@ -6,8 +6,7 @@ const ts = require('typescript');
|
|||||||
const path = require('node:path');
|
const path = require('node:path');
|
||||||
// The provider settings ios-preview-controller hands the voice bridge.
|
// The provider settings ios-preview-controller hands the voice bridge.
|
||||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||||
getVoiceTranscriptionLanguage: () => '',
|
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||||
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
|
||||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||||
function load(file, context) {
|
function load(file, context) {
|
||||||
@@ -169,7 +168,6 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
|
|||||||
let cachedFont = 'Light';
|
let cachedFont = 'Light';
|
||||||
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
||||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||||
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
|
|
||||||
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
||||||
setTimeout: fn => tasks.push(fn),
|
setTimeout: fn => tasks.push(fn),
|
||||||
});
|
});
|
||||||
@@ -386,7 +384,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
|||||||
recordTo: name => { bridge.recording = name; },
|
recordTo: name => { bridge.recording = name; },
|
||||||
setRawSession: session => { bridge.rawSession = session; } };
|
setRawSession: session => { bridge.rawSession = session; } };
|
||||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
|
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||||
saveTranscribeAudioSetting: { get: () => true } };
|
saveTranscribeAudioSetting: { get: () => true } };
|
||||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ const generator = require('../scripts/generate-pebble-icons.cjs');
|
|||||||
function setup(notifications) {
|
function setup(notifications) {
|
||||||
const load = loader({}, {
|
const load = loader({}, {
|
||||||
'../g2/ancs-client': { AncsClient: class {}, ANCS_CONNECT_MESSAGE: '' },
|
'../g2/ancs-client': { AncsClient: class {}, ANCS_CONNECT_MESSAGE: '' },
|
||||||
'./notification-sources': { rememberNotificationSources() {} },
|
'./notification-sources': { rememberNotificationSources() {}, shouldShowNotificationOnGlasses: packageName => packageName !== "muted" },
|
||||||
});
|
});
|
||||||
const icons = load('app/native/notification-icons.ios.ts');
|
const icons = load('app/native/notification-icons.ios.ts');
|
||||||
icons.bindIosNotifications({ read: () => notifications });
|
icons.bindIosNotifications({ read: () => notifications });
|
||||||
@@ -106,3 +106,11 @@ test('the icon generator inverts Pebble art: black ink lights up, white fills hi
|
|||||||
const decoded = loader({}, { './pebble-icons-data': loader({}, {})(dataFile) })('app/graphics/pebble-icons.ts').pebbleIcon('probe');
|
const decoded = loader({}, { './pebble-icons-data': loader({}, {})(dataFile) })('app/graphics/pebble-icons.ts').pebbleIcon('probe');
|
||||||
assert.deepEqual([...decoded.pixels], [...pixels]);
|
assert.deepEqual([...decoded.pixels], [...pixels]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('muted sources are hidden before list and icon limits without removing phone notifications', () => {
|
||||||
|
const notifications = [notification('hidden', 'muted', '0'), notification('shown', 'allowed', '0')];
|
||||||
|
const { icons } = setup(notifications);
|
||||||
|
assert.deepEqual([...icons.readVisibleNotifications(1)].map(n => n.key), ['shown']);
|
||||||
|
assert.deepEqual([...icons.readActiveNotificationIcons(1, false).keys], ['shown']);
|
||||||
|
assert.equal(icons.readActiveNotifications(10).length, 2);
|
||||||
|
});
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package com.faceclaw.app
|
|
||||||
|
|
||||||
import kotlin.test.Test
|
|
||||||
import kotlin.test.assertEquals
|
|
||||||
import kotlin.test.assertFalse
|
|
||||||
import kotlin.test.assertTrue
|
|
||||||
|
|
||||||
class WhisperModelFilesTest {
|
|
||||||
@Test fun existingEnglishFilesRemainUsable() {
|
|
||||||
for (language in listOf(null, "en", " EN ")) {
|
|
||||||
val model = WhisperModelFiles.forLanguage(language)
|
|
||||||
assertTrue(model.englishOnly)
|
|
||||||
assertEquals("en", model.language)
|
|
||||||
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
|
|
||||||
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
|
|
||||||
for (language in listOf("fr", "de", "ja", " FR ")) {
|
|
||||||
val model = WhisperModelFiles.forLanguage(language)
|
|
||||||
assertFalse(model.englishOnly)
|
|
||||||
assertEquals(language.trim().lowercase(), model.language)
|
|
||||||
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
|
|
||||||
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
|
|
||||||
val model = WhisperModelFiles.forLanguage("")
|
|
||||||
assertFalse(model.englishOnly)
|
|
||||||
assertEquals("", model.language)
|
|
||||||
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -70,6 +70,7 @@ function ui(fontSize = 12) {
|
|||||||
const native = {
|
const native = {
|
||||||
ALL_NOTIFICATIONS: 0x7fffffff,
|
ALL_NOTIFICATIONS: 0x7fffffff,
|
||||||
readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; },
|
readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; },
|
||||||
|
readVisibleNotifications: () => { prefs.rememberNotificationSources(active); return active.filter(n => prefs.shouldShowNotificationOnGlasses(n.packageName)); },
|
||||||
dismissNotification: () => { dismissals++; active = []; return true; },
|
dismissNotification: () => { dismissals++; active = []; return true; },
|
||||||
readNotificationIconByKey: () => ({ icon: null, stale: false }),
|
readNotificationIconByKey: () => ({ icon: null, stale: false }),
|
||||||
};
|
};
|
||||||
@@ -165,6 +166,7 @@ test('cancel and back dismiss the popup without changing source preferences', as
|
|||||||
test('filter toggles persist after notifications disappear and discovers new sources while open', async () => {
|
test('filter toggles persist after notifications disappear and discovers new sources while open', async () => {
|
||||||
const app = ui();
|
const app = ui();
|
||||||
app.paint(app.filter);
|
app.paint(app.filter);
|
||||||
|
for (let i = 0; i < 3; i++) await app.input(app.filter, 'scroll-down');
|
||||||
await app.input(app.filter, 'click');
|
await app.input(app.filter, 'click');
|
||||||
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
|
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
|
||||||
app.active([{ packageName: 'com.alpha', appName: 'Alpha' }]);
|
app.active([{ packageName: 'com.alpha', appName: 'Alpha' }]);
|
||||||
@@ -177,6 +179,68 @@ test('filter toggles persist after notifications disappear and discovers new sou
|
|||||||
assert.equal(app.prefs.shouldShowNotificationOnGlasses('com.alpha'), true);
|
assert.equal(app.prefs.shouldShowNotificationOnGlasses('com.alpha'), true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('default persists without overwriting explicit choices, including legacy saved sources', () => {
|
||||||
|
const settings = new Map([['notifications.sources', JSON.stringify([
|
||||||
|
{ ...mail, showOnGlasses: true },
|
||||||
|
{ packageName: 'muted', appName: 'Muted', showOnGlasses: false },
|
||||||
|
])]]);
|
||||||
|
let prefs = store(settings);
|
||||||
|
prefs.rememberNotificationSources([{ packageName: 'discovered', appName: 'Discovered' }]);
|
||||||
|
prefs.setNewNotificationAppsEnabledByDefault(false);
|
||||||
|
prefs = store(settings);
|
||||||
|
assert.equal(prefs.newNotificationAppsEnabledByDefault(), false);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('muted'), false);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('discovered'), false);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
|
||||||
|
prefs.rememberNotificationSources([{ packageName: 'future', appName: 'Future' }]);
|
||||||
|
prefs.setNotificationSourceEnabled({ packageName: 'future', appName: 'Future' }, true);
|
||||||
|
prefs.setNewNotificationAppsEnabledByDefault(true);
|
||||||
|
prefs.setNewNotificationAppsEnabledByDefault(false);
|
||||||
|
assert.equal(store(settings).shouldShowNotificationOnGlasses('future'), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('bulk choices persist for known sources without changing the default for future sources', () => {
|
||||||
|
const settings = new Map();
|
||||||
|
let prefs = store(settings);
|
||||||
|
prefs.rememberNotificationSources([mail]);
|
||||||
|
prefs.setAllNotificationSourcesEnabled(false);
|
||||||
|
prefs = store(settings);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), true);
|
||||||
|
prefs.setNewNotificationAppsEnabledByDefault(false);
|
||||||
|
prefs.setAllNotificationSourcesEnabled(true);
|
||||||
|
prefs = store(settings);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
|
||||||
|
prefs.rememberNotificationSources([{ packageName: 'future', appName: 'Future' }]);
|
||||||
|
assert.equal(prefs.shouldShowNotificationOnGlasses('future'), false);
|
||||||
|
prefs.setAllNotificationSourcesEnabled(false);
|
||||||
|
assert.ok(prefs.readNotificationSources().every(source => !source.showOnGlasses));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('filter controls work with an empty list and keep selection as sources arrive', async () => {
|
||||||
|
const app = ui();
|
||||||
|
app.active([]);
|
||||||
|
const labels = app.paint(app.filter).texts.map(({ text }) => text);
|
||||||
|
assert.ok(labels.includes('New apps enabled by default'));
|
||||||
|
assert.ok(labels.includes('Enable all'));
|
||||||
|
assert.ok(labels.includes('Disable all'));
|
||||||
|
await app.input(app.filter, 'click');
|
||||||
|
assert.equal(app.prefs.newNotificationAppsEnabledByDefault(), false);
|
||||||
|
app.active([mail]);
|
||||||
|
app.paint(app.filter);
|
||||||
|
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
|
||||||
|
await app.input(app.filter, 'scroll-down');
|
||||||
|
await app.input(app.filter, 'click');
|
||||||
|
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), true);
|
||||||
|
assert.equal(app.prefs.newNotificationAppsEnabledByDefault(), false);
|
||||||
|
await app.input(app.filter, 'scroll-down');
|
||||||
|
await app.input(app.filter, 'click');
|
||||||
|
assert.equal(app.prefs.shouldShowNotificationOnGlasses(mail.packageName), false);
|
||||||
|
assert.equal(app.dismissals(), 0);
|
||||||
|
});
|
||||||
|
|
||||||
test('popup action wraps fully and stays visible with small and large fonts', async () => {
|
test('popup action wraps fully and stays visible with small and large fonts', async () => {
|
||||||
for (const size of [12, 20]) {
|
for (const size of [12, 20]) {
|
||||||
const app = ui(size);
|
const app = ui(size);
|
||||||
@@ -254,3 +318,33 @@ test('controller filters before waking or opening a popup, and still refreshes t
|
|||||||
elapse();
|
elapse();
|
||||||
assert.equal(popups, 1);
|
assert.equal(popups, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Android display filters before limits and reapplies preferences to cached tray icons', () => {
|
||||||
|
const prefs = store();
|
||||||
|
const active = [{ ...mail, key: 'hidden' }, { packageName: 'allowed', appName: 'Allowed', key: 'shown' }];
|
||||||
|
const file = ts.createSourceFile('icons.ts', source('app/native/notification-icons.ts'), ts.ScriptTarget.Latest, true);
|
||||||
|
const functions = file.statements.filter(node => ts.isFunctionDeclaration(node) &&
|
||||||
|
['readVisibleNotifications', 'cachedIconsResult'].includes(node.name?.text)).map(node => node.getText(file));
|
||||||
|
const { readVisibleNotifications, cachedIconsResult } = evaluate(functions.join('\n') + '\nexport { cachedIconsResult };', null, {
|
||||||
|
ALL_NOTIFICATIONS: 0x7fffffff,
|
||||||
|
readActiveNotifications: () => { prefs.rememberNotificationSources(active); return active; },
|
||||||
|
shouldShowNotificationOnGlasses: prefs.shouldShowNotificationOnGlasses,
|
||||||
|
cachedKeys: ['hidden', 'shown'],
|
||||||
|
cachedPackages: new Map(active.map(n => [n.key, n.packageName])),
|
||||||
|
cachedIcons: [{ clone: () => 'mail-icon' }, { clone: () => 'allowed-icon' }],
|
||||||
|
});
|
||||||
|
prefs.setNotificationSourceEnabled(mail, false);
|
||||||
|
assert.deepEqual(Array.from(readVisibleNotifications(1), n => n.key), ['shown']);
|
||||||
|
for (const stale of [false, true]) {
|
||||||
|
const result = cachedIconsResult(1, stale);
|
||||||
|
assert.deepEqual(Array.from(result.keys), ['shown']);
|
||||||
|
assert.deepEqual(Array.from(result.icons), ['allowed-icon']);
|
||||||
|
assert.equal(result.stale, stale);
|
||||||
|
}
|
||||||
|
prefs.setAllNotificationSourcesEnabled(false);
|
||||||
|
assert.equal(cachedIconsResult(10, false).keys.length, 0);
|
||||||
|
assert.equal(readVisibleNotifications(50).length, 0);
|
||||||
|
prefs.setNotificationSourceEnabled(mail, true);
|
||||||
|
assert.deepEqual(Array.from(cachedIconsResult(1, false).keys), ['hidden']);
|
||||||
|
assert.equal(active.length, 2);
|
||||||
|
});
|
||||||
@@ -28,7 +28,6 @@
|
|||||||
"../app/ui/menu-core.ts",
|
"../app/ui/menu-core.ts",
|
||||||
"../app/phone-ui/phone-gestures.ts",
|
"../app/phone-ui/phone-gestures.ts",
|
||||||
"../app/native/java-direct-buffer.ts",
|
"../app/native/java-direct-buffer.ts",
|
||||||
"../app/native/transcription-language.ts",
|
|
||||||
"../app/native/notification-types.ts",
|
"../app/native/notification-types.ts",
|
||||||
"../app/g2/firmware-compat.ts",
|
"../app/g2/firmware-compat.ts",
|
||||||
"../app/g2/glasses-hardware-identity.ts",
|
"../app/g2/glasses-hardware-identity.ts",
|
||||||
|
|||||||
@@ -163,8 +163,7 @@ function androidBridge() {
|
|||||||
starts: 0,
|
starts: 0,
|
||||||
setListener(listener) { this.listener = listener; },
|
setListener(listener) { this.listener = listener; },
|
||||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
|
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||||
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
|
|
||||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||||
stop() { this.active = false; },
|
stop() { this.active = false; },
|
||||||
isCapturing() { return this.active; },
|
isCapturing() { return this.active; },
|
||||||
@@ -172,8 +171,7 @@ function androidBridge() {
|
|||||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||||
'./cloud-stt-provider': { createCloudSttClient: () => null,
|
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||||
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
|
|
||||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||||
FaceclawVoiceController: function () { return controller; },
|
FaceclawVoiceController: function () { return controller; },
|
||||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||||
@@ -181,26 +179,6 @@ function androidBridge() {
|
|||||||
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
||||||
}
|
}
|
||||||
|
|
||||||
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
|
|
||||||
const { bridge, controller } = androidBridge();
|
|
||||||
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
|
|
||||||
assert.equal(controller.modelKind, 'whisper');
|
|
||||||
assert.equal(controller.language, 'fr');
|
|
||||||
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
|
|
||||||
assert.equal(controller.starts, 1);
|
|
||||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
|
|
||||||
assert.equal(controller.language, '');
|
|
||||||
assert.equal(controller.starts, 2);
|
|
||||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
|
||||||
// An omitted language preserves the native bridge's original English behavior.
|
|
||||||
assert.equal(controller.language, 'en');
|
|
||||||
assert.equal(controller.starts, 3);
|
|
||||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
|
|
||||||
assert.equal(controller.language, 'en');
|
|
||||||
assert.equal(controller.starts, 3);
|
|
||||||
bridge.stop();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||||
const { bridge, controller } = androidBridge();
|
const { bridge, controller } = androidBridge();
|
||||||
const options = { provider: 'onboard-whisper' };
|
const options = { provider: 'onboard-whisper' };
|
||||||
|
|||||||
@@ -1,146 +0,0 @@
|
|||||||
const test = require('node:test');
|
|
||||||
const assert = require('node:assert/strict');
|
|
||||||
const fs = require('node:fs');
|
|
||||||
const path = require('node:path');
|
|
||||||
const vm = require('node:vm');
|
|
||||||
const ts = require('typescript');
|
|
||||||
|
|
||||||
function load(file, requireModule = () => ({}), globals = {}) {
|
|
||||||
const exports = {};
|
|
||||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
|
||||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
|
||||||
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
|
|
||||||
return exports;
|
|
||||||
}
|
|
||||||
const languages = load('app/native/transcription-language.ts');
|
|
||||||
function settings(values = new Map()) {
|
|
||||||
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
|
|
||||||
: name.includes('settings-store') ? {
|
|
||||||
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
|
||||||
setStringSetting: (key, value) => values.set(key, value),
|
|
||||||
getBooleanSetting: (_key, fallback) => fallback,
|
|
||||||
onSettingsStoreChanged() {},
|
|
||||||
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
|
|
||||||
}
|
|
||||||
|
|
||||||
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
|
|
||||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
|
||||||
const s = settings(values);
|
|
||||||
assert.equal(s.voiceLanguageSetting.get(), 'default');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
|
||||||
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
|
|
||||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
|
||||||
let s = settings(values);
|
|
||||||
s.voiceLanguageSetting.set('fr');
|
|
||||||
s = settings(values);
|
|
||||||
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
|
|
||||||
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
|
|
||||||
}
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
|
|
||||||
s.voiceLanguageSetting.set('auto');
|
|
||||||
s = settings(values);
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
|
||||||
s.voiceLanguageSetting.set('default');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('corrupt language preferences revert to the compatible provider defaults', () => {
|
|
||||||
const s = settings(new Map([['voice.language', 'invalid']]));
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
|
||||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
|
|
||||||
const s = settings();
|
|
||||||
s.voiceLanguageSetting.set('fr');
|
|
||||||
s.voiceProviderSetting.set('whisper');
|
|
||||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
|
||||||
for (const [file, name, method] of [
|
|
||||||
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
|
||||||
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
|
||||||
]) {
|
|
||||||
const host = { continuousClients: new Map() };
|
|
||||||
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
|
|
||||||
// Run each real capture-options method with only its native dependencies stubbed.
|
|
||||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
|
||||||
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
|
||||||
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
|
|
||||||
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
|
||||||
const capture = loadBody(member.body.getText(tree), s, host);
|
|
||||||
assert.equal(capture().language, 'fr');
|
|
||||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
|
||||||
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
|
||||||
assert.equal(capture().provider, 'soniox');
|
|
||||||
assert.equal(capture().language, 'ja');
|
|
||||||
host.continuousClients.clear();
|
|
||||||
assert.equal(capture().language, 'fr');
|
|
||||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
function loadBody(body, s, host) {
|
|
||||||
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
|
|
||||||
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
test('multilingual downloads and deletion leave an existing English model intact', () => {
|
|
||||||
const files = new Map(), downloads = [];
|
|
||||||
const asr = load('app/native/asr-model.ts', () => ({
|
|
||||||
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
|
||||||
}), {
|
|
||||||
java: { io: { File: class {
|
|
||||||
constructor(name) { this.name = name; }
|
|
||||||
exists() { return files.has(this.name); }
|
|
||||||
length() { return files.get(this.name) ?? 0; }
|
|
||||||
delete() { return files.delete(this.name); }
|
|
||||||
} } },
|
|
||||||
com: { faceclaw: { app: {
|
|
||||||
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
|
||||||
FaceclawModelDownloader: class {
|
|
||||||
constructor(url, destination, hash, size, listener) {
|
|
||||||
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
|
||||||
}
|
|
||||||
start() {}
|
|
||||||
cancel() {}
|
|
||||||
},
|
|
||||||
} } },
|
|
||||||
});
|
|
||||||
const english = asr.ASR_MODELS['whisper-base-en'];
|
|
||||||
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
|
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
|
||||||
asr.startAsrModelDownload('whisper-base');
|
|
||||||
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
|
|
||||||
for (let index = 0; index < 3; index++) {
|
|
||||||
const download = downloads[index];
|
|
||||||
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
|
|
||||||
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
|
||||||
files.set(download.destination, download.size);
|
|
||||||
download.listener.onDone();
|
|
||||||
}
|
|
||||||
assert.equal(downloads.length, 3);
|
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base'), true);
|
|
||||||
asr.deleteAsrModel('whisper-base');
|
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
|
||||||
const values = new Map();
|
|
||||||
let s = settings(values);
|
|
||||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
|
||||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
|
||||||
s.voiceLanguageSetting.set('fr');
|
|
||||||
s = settings(values);
|
|
||||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
|
||||||
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
|
||||||
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
|
||||||
});
|
|
||||||
Reference in new issue
Block a user