Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a80d6697b5
|
||
|
|
0b9e4cb367
|
No files matched your search
@@ -25,18 +25,21 @@ internal object AndroidSpeechEngines {
|
||||
private const val FEATURE_DIM = 80
|
||||
|
||||
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
||||
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
|
||||
fun recognizerConfig(
|
||||
modelDir: File, kind: VoiceModelKind,
|
||||
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
|
||||
): OfflineRecognizerConfig {
|
||||
val modelConfig = OfflineModelConfig.builder()
|
||||
.setNumThreads(1)
|
||||
if (kind == VoiceModelKind.WHISPER) {
|
||||
modelConfig
|
||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
|
||||
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
|
||||
.setLanguage("en")
|
||||
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
|
||||
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
|
||||
.setLanguage(whisper.language)
|
||||
.setTask("transcribe")
|
||||
.build())
|
||||
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
|
||||
.setTokens(File(modelDir, whisper.tokens).absolutePath)
|
||||
} else {
|
||||
modelConfig
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
|
||||
@@ -32,17 +32,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
)
|
||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||
// Whisper re-encodes the whole buffer on every call, so the shared session
|
||||
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
|
||||
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
|
||||
private val ASR_WHISPER_MODEL_FILES = arrayOf(
|
||||
"base.en-encoder.int8.onnx",
|
||||
"base.en-decoder.int8.onnx",
|
||||
"base.en-tokens.txt"
|
||||
)
|
||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||
// (endpointing, transcript pacing) is tuned for.
|
||||
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
||||
@@ -84,6 +73,7 @@ class FaceclawVoiceController(context: Context) {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
@Volatile
|
||||
private var communicator: FaceclawBleCommunicator? = null
|
||||
private var whisperModel = WhisperModelFiles.forLanguage(null)
|
||||
|
||||
private val host = object : VoiceCaptureHost {
|
||||
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
||||
@@ -122,7 +112,7 @@ class FaceclawVoiceController(context: Context) {
|
||||
|
||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
||||
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
|
||||
}
|
||||
|
||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
||||
@@ -168,6 +158,11 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setOnboardModelKind(kind)
|
||||
}
|
||||
|
||||
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
|
||||
fun setOnboardLanguage(language: String?) {
|
||||
whisperModel = WhisperModelFiles.forLanguage(language)
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
@@ -263,8 +258,8 @@ class FaceclawVoiceController(context: Context) {
|
||||
*/
|
||||
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
||||
val whisper = kind == VoiceModelKind.WHISPER
|
||||
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
|
||||
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
|
||||
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
||||
for (fileName in fileNames) {
|
||||
val file = File(modelDir, fileName)
|
||||
|
||||
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -121,7 +121,6 @@ export type DashboardSnapshot = {
|
||||
* glasses, and this says which harmless thing is actually going on.
|
||||
*/
|
||||
displayPreviewMessage: string;
|
||||
silentModeFeedback: string;
|
||||
activeTextSettingId: string | null;
|
||||
activeTextEditorTitle: string;
|
||||
activeTextSettingTitle: string;
|
||||
@@ -295,9 +294,6 @@ class DashboardController {
|
||||
/** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */
|
||||
private displayPreviewCrop: ScreenRect | null = null;
|
||||
private silentMode = false;
|
||||
private silentModeFeedback = "";
|
||||
private silentModeFeedbackGeneration = 0;
|
||||
private silentModeFeedbackTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
// Brightness value last pushed to the glasses; see pushBrightness.
|
||||
private lastPushedBrightness: string | null = null;
|
||||
// Last battery level the glasses reported, kept across disconnects so a
|
||||
@@ -487,6 +483,7 @@ class DashboardController {
|
||||
this.applyVerticalPositionIfChanged();
|
||||
this.applyAppLayoutsIfChanged();
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
this.retargetContinuousCapture();
|
||||
this.syncLockScreenSettingIfChanged();
|
||||
// A Preview color or scaling change should show on the mirror at once,
|
||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||
@@ -1146,7 +1143,6 @@ class DashboardController {
|
||||
status: this.status,
|
||||
displayPreview: this.displayPreview,
|
||||
displayPreviewMessage: this.displayPreviewMessage(),
|
||||
silentModeFeedback: this.silentModeFeedback,
|
||||
activeTextSettingId: primaryTextSetting?.id ?? null,
|
||||
activeTextEditorTitle: this.activeTextEditorTitle,
|
||||
activeTextSettingTitle: primaryTextSetting?.editorTitle ?? "",
|
||||
@@ -1186,43 +1182,6 @@ class DashboardController {
|
||||
* charging and silent mode both make the display unavailable while BLE stays
|
||||
* up, and a battery that just ran out simply stops answering.
|
||||
*/
|
||||
private handleSilentMode(silent: boolean): void {
|
||||
if (this.silentMode === silent) return;
|
||||
this.silentMode = silent;
|
||||
const generation = ++this.silentModeFeedbackGeneration;
|
||||
if (this.silentModeFeedbackTimer !== null) clearTimeout(this.silentModeFeedbackTimer);
|
||||
this.silentModeFeedbackTimer = null;
|
||||
this.silentModeFeedback = silent
|
||||
? "Silent mode enabled — hold both touchpads to exit"
|
||||
: "Silent mode disabled";
|
||||
if (!silent) {
|
||||
void this.showSilentModeExit(generation).catch(error => {
|
||||
this.appendLog(`silent-mode exit feedback failed: ${this.formatError(error)}`);
|
||||
});
|
||||
this.silentModeFeedbackTimer = setTimeout(() => {
|
||||
this.silentModeFeedbackTimer = null;
|
||||
this.silentModeFeedback = "";
|
||||
this.emit();
|
||||
}, 5000);
|
||||
}
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private async showSilentModeExit(generation: number): Promise<void> {
|
||||
if (this.phase !== "connected" || this.silentMode || this.glassesLocked) return;
|
||||
// The firmware powered the lenses down, and Faceclaw may have gone to
|
||||
// sleep/suspended EvenHub meanwhile. Reusing the ordinary wake barrier is
|
||||
// essential: rendering an alert into the retained surface alone is invisible.
|
||||
this.cancelEvenHubSuspendTimer();
|
||||
shell.wake("sidebar");
|
||||
const ready = await this.ensureEvenHubSessionActive();
|
||||
if (!ready || generation !== this.silentModeFeedbackGeneration || this.silentMode ||
|
||||
this.phase !== "connected" || this.glassesLocked) return;
|
||||
shell.showAlert("Silent mode disabled");
|
||||
shell.foregroundWindow()?.requestRender();
|
||||
this.requestShellRender();
|
||||
}
|
||||
|
||||
private displayPreviewMessage(): string {
|
||||
if (this.phase === "charging") {
|
||||
return chargingPreviewMessage(this.glassesDisplayState());
|
||||
@@ -1646,7 +1605,9 @@ class DashboardController {
|
||||
});
|
||||
});
|
||||
this.offSilentMode = communicator.onSilentMode((silent) => {
|
||||
this.handleSilentMode(silent);
|
||||
if (this.silentMode === silent) return;
|
||||
this.silentMode = silent;
|
||||
this.emit();
|
||||
});
|
||||
this.offWearState = communicator.onWearState((wearing) => {
|
||||
this.handleWearState(wearing);
|
||||
@@ -2237,16 +2198,16 @@ class DashboardController {
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
// Otherwise use the shared transcription-language preference.
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
@@ -3024,10 +2985,6 @@ class DashboardController {
|
||||
// Kept across "connecting": silent mode blocks app launches, so it can
|
||||
// itself cause the reconnect churn, and Java re-reports it either way.
|
||||
this.silentMode = false;
|
||||
this.silentModeFeedbackGeneration++;
|
||||
if (this.silentModeFeedbackTimer !== null) clearTimeout(this.silentModeFeedbackTimer);
|
||||
this.silentModeFeedbackTimer = null;
|
||||
this.silentModeFeedback = "";
|
||||
}
|
||||
this.emit();
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
@@ -416,6 +416,7 @@ export class IosPreviewController {
|
||||
const phone = readPhoneBatteryState()
|
||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.syncContinuousCapture()
|
||||
this.syncLockSetting()
|
||||
this.pushBrightness()
|
||||
this.relayout()
|
||||
@@ -1198,14 +1199,13 @@ export class IosPreviewController {
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
* every capture while it is open; otherwise use the shared language preference.
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
|
||||
+32
-2
@@ -9,7 +9,7 @@ declare const java: any;
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Two models, same shape:
|
||||
* Downloadable models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
@@ -29,12 +29,17 @@ declare const java: any;
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
|
||||
* speech and auto-detection; English-only downloads remain untouched.
|
||||
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
||||
* vocabulary hash computed locally. URLs pin the source revision.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
@@ -105,6 +110,30 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
|
||||
"whisper-base": {
|
||||
label: "Whisper (multilingual, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-int8",
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
|
||||
files: [
|
||||
{
|
||||
name: "base-encoder.int8.onnx",
|
||||
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base-decoder.int8.onnx",
|
||||
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
|
||||
sizeBytes: 130672026,
|
||||
},
|
||||
{
|
||||
name: "base-tokens.txt",
|
||||
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
||||
sizeBytes: 816730,
|
||||
},
|
||||
],
|
||||
totalBytes: 160609290,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -129,6 +158,7 @@ function freshRuntime(): ModelRuntime {
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
"whisper-base": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
@@ -14,11 +14,12 @@ export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Cloud providers only.
|
||||
* detect the language. Also used by on-device Whisper.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
@@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
whisper: {
|
||||
name: "OpenAI",
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
||||
},
|
||||
"whisper-server": {
|
||||
name: "Whisper server",
|
||||
@@ -76,8 +77,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
||||
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
|
||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
||||
* on the Realtime API, with a configurable transcription model).
|
||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||
*
|
||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||
@@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
const MODEL_ID = "gpt-realtime-whisper";
|
||||
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
||||
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: SocketConnection | null = null;
|
||||
@@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
this.options.onError(message.error?.code === "model_not_found"
|
||||
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
||||
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
@@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(language: string): object {
|
||||
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -209,7 +212,7 @@ function sessionConfig(language: string): object {
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
transcription: { model, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/** Languages supported by both Whisper backends, in picker order. */
|
||||
export const TRANSCRIPTION_LANGUAGES = {
|
||||
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
|
||||
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
|
||||
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
|
||||
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
|
||||
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
|
||||
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
|
||||
|
||||
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
|
||||
export function resolveTranscriptionLanguage(
|
||||
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
||||
): string {
|
||||
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
||||
if (provider === "onboard") return "";
|
||||
if (choice === "default") {
|
||||
if (provider === "onboard-whisper") return "en";
|
||||
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
|
||||
}
|
||||
return choice === "auto" ? "" : choice;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { OpenAiSttModel } from "./openai-stt";
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
@@ -24,7 +25,7 @@ export type VoiceControlState = {
|
||||
detail: string;
|
||||
};
|
||||
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
@@ -54,11 +55,12 @@ export type PushToTalkOptions = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
@@ -396,6 +398,7 @@ export class FaceclawVoiceControlBridge {
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
this.controller?.setOnboardLanguage(options.language ?? "en");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
|
||||
@@ -74,9 +74,6 @@
|
||||
class="ble-bandwidth-indicator"
|
||||
horizontalAlignment="center"
|
||||
verticalAlignment="bottom" />
|
||||
<Label text="{{ silentModeFeedback }}" visibility="{{ silentModeFeedbackVisibility }}"
|
||||
textWrap="true" horizontalAlignment="center" verticalAlignment="top"
|
||||
margin="12" padding="12" backgroundColor="#333333" color="white" borderRadius="8" />
|
||||
<ScrollView
|
||||
class="warnings-modal"
|
||||
visibility="{{ warningsModalVisibility }}"
|
||||
|
||||
@@ -37,7 +37,6 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
private _status = "Disconnected.";
|
||||
private _displayPreview: ImageSource | null = null;
|
||||
private _displayPreviewMessage = "";
|
||||
private _silentModeFeedback = "";
|
||||
private _layoutOrientation: LayoutOrientation = this.readLayoutOrientation();
|
||||
private _landscapePreviewWidth = 0;
|
||||
private _landscapePreviewHeight = 0;
|
||||
@@ -91,7 +90,6 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
this.previewMode = snapshot.previewMode;
|
||||
this.displayPreview = snapshot.displayPreview;
|
||||
this.displayPreviewMessage = snapshot.displayPreviewMessage;
|
||||
this.silentModeFeedback = snapshot.silentModeFeedback ?? "";
|
||||
this.phase = snapshot.phase;
|
||||
this.batteryLine = formatBatteryLine(
|
||||
snapshot.batteries,
|
||||
@@ -218,17 +216,6 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
}
|
||||
}
|
||||
|
||||
get silentModeFeedback(): string { return this._silentModeFeedback; }
|
||||
set silentModeFeedback(value: string) {
|
||||
if (this._silentModeFeedback === value) return;
|
||||
this._silentModeFeedback = value;
|
||||
this.notifyPropertyChange("silentModeFeedback", value);
|
||||
this.notifyPropertyChange("silentModeFeedbackVisibility", this.silentModeFeedbackVisibility);
|
||||
}
|
||||
get silentModeFeedbackVisibility(): "visible" | "collapse" {
|
||||
return this._silentModeFeedback ? "visible" : "collapse";
|
||||
}
|
||||
|
||||
get hasDisplayPreview(): boolean {
|
||||
return this._displayPreview !== null;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { OpenAiSttModel } from "../native/openai-stt";
|
||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
|
||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -1136,9 +1138,35 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||
});
|
||||
|
||||
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
||||
id: "openai-transcription-model",
|
||||
label: "OpenAI transcription model",
|
||||
storageKey: "voice.openAiTranscriptionModel",
|
||||
defaultValue: "gpt-realtime-whisper",
|
||||
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
||||
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
||||
});
|
||||
|
||||
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
||||
id: "voice-language",
|
||||
label: "Transcription language",
|
||||
storageKey: "voice.language",
|
||||
defaultValue: "default",
|
||||
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
|
||||
formatValue: (value) => value === "default" ? "Provider default"
|
||||
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
||||
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
||||
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
|
||||
});
|
||||
|
||||
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
||||
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
|
||||
}
|
||||
|
||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
id: "whisper-server-language",
|
||||
label: "Whisper language",
|
||||
label: "Whisper server language",
|
||||
storageKey: "voice.whisperServerLanguage",
|
||||
defaultValue: "",
|
||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||
@@ -1146,7 +1174,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
formatValue: (value) => value || "Auto-detect",
|
||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||
description:
|
||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
|
||||
});
|
||||
|
||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -63,6 +63,7 @@ import {
|
||||
mapboxApiKeySetting,
|
||||
mirrorTouchSetting,
|
||||
openAiApiKeySetting,
|
||||
openAiTranscriptionModelSetting,
|
||||
previewColorSetting,
|
||||
previewScalingSetting,
|
||||
phoneRotationSetting,
|
||||
@@ -87,6 +88,7 @@ import {
|
||||
toggleSettingMenuItem,
|
||||
useMicControlSetting,
|
||||
voiceProviderSetting,
|
||||
voiceLanguageSetting,
|
||||
screenTimeoutSetting,
|
||||
wakeToLastAppSetting,
|
||||
wakeWordActionSetting,
|
||||
@@ -143,9 +145,11 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
enumSettingMenuItem(voiceLanguageSetting),
|
||||
...whisperServerMenuItems(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
asrModelMenuItem("whisper-base"),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -284,7 +288,7 @@ function settingsSections(): SettingsSection[] {
|
||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||
item.label !== screenTimeoutSetting.label) };
|
||||
return section;
|
||||
@@ -527,6 +531,7 @@ function asrModelStatusText(id: AsrModelId): string {
|
||||
*/
|
||||
function whisperServerMenuItems(): MenuItem[] {
|
||||
return [
|
||||
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
||||
textSettingMenuItem(whisperServerUrlSetting),
|
||||
textSettingMenuItem(whisperServerModelSetting),
|
||||
textSettingMenuItem(whisperServerLanguageSetting),
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
# Silent-mode feedback
|
||||
|
||||
The firmware blanks the lenses and stops ordinary input while silent mode is active. App code cannot display an activation popup on lenses that the firmware has switched off. The phone now shows a persistent enabled banner with the exit gesture, and a five-second disabled banner on exit. On exit, unlocked glasses wake through the normal screen/EvenHub barrier before the alert popup is rendered. This also handles the shell falling asleep during a long silent period. Locked glasses keep the phone feedback without being woken. A generation check suppresses stale notices after re-entry or disconnect. Duplicate state reports do not repeat feedback, stale timers are cancelled, and disconnect clears the notice.
|
||||
|
||||
Silent-off firmware pushes are not reliable. The existing fallback read used the five-minute battery cadence. While silent mode is known to be on, settings are now polled every two seconds when the session is ready and the message queues are idle. Normal battery cadence and shutdown guards are retained. This bounds ordinary missing-push recovery without assuming a touch gesture always means silent mode ended.
|
||||
|
||||
Independent JavaScript suite: 1065 passed, 2 skipped. TypeScript and changed-source lint passed. Controller tests cover enabled/disabled notices, duplicates, timer cancellation, disconnect and sleeping-display recovery, locked displays, wake-barrier failure and stale async completion. A Kotlin session test checks the two-second silent cadence, normal cadence and shutdown guard. Kotlin compilation and hardware checks are tracked with the combined integration validation.
|
||||
|
||||
Hardware validation remains pending: both transitions via the temple gesture, missing exit push, phone background/foreground, lenses that are awake/asleep/locked, reconnect and charging. No firmware was flashed.
|
||||
|
||||
Follow-up after device-log inspection: both silent transitions arrived, but the exit could occur after the normal idle timeout. Exit now cancels pending suspension, wakes the shell, awaits the existing EvenHub/screen wake barrier and only then shows the notice. It resets idle activity through the ordinary wake path. Re-entry, disconnect, locking or a failed resume suppress stale asynchronous notices. Activation still turns the lenses off in firmware; phone feedback does not claim a lens-side activation popup.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Whisper transcription languages
|
||||
|
||||
Settings > Voice > Transcription language selects French, English, 27 other
|
||||
languages, or auto-detection. It applies to ordinary dictation and continuous
|
||||
transcription using OpenAI, on-device Whisper, and the other cloud providers.
|
||||
Translate retains its own spoken-language preference. Moonshine remains
|
||||
English-only; Apple's on-device speech recognizer retains its own locale.
|
||||
|
||||
The initial **Provider default** preserves existing installations: English for
|
||||
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
|
||||
Whisper server language for self-hosted Whisper. Selecting another shared
|
||||
language overrides the server language without deleting it. Returning to Provider
|
||||
default restores it. Settings persist through the existing settings store.
|
||||
|
||||
For French or auto-detection with **On-device (Whisper)**, download
|
||||
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
|
||||
English uses the existing English-only base model; neither model is deleted or
|
||||
downloaded automatically. A missing model uses the existing “Voice model not
|
||||
downloaded” status. Language changes restart active continuous transcription;
|
||||
other captures use the new language when started.
|
||||
|
||||
## Software validation
|
||||
|
||||
- JavaScript regression tests exercise stored settings/reload, compatible
|
||||
defaults, French and auto-detection, Translate overrides in Android/iOS capture
|
||||
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
|
||||
- Kotlin tests exercise English/French/other/automatic model selection and file
|
||||
names. Android release compilation checks the actual sherpa-onnx bridge.
|
||||
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
|
||||
the existing resumable downloader. Encoder/decoder hashes and sizes come from
|
||||
repository LFS metadata; the vocabulary hash was computed from its contents.
|
||||
|
||||
## Device validation still required
|
||||
|
||||
With both phone microphone preview and glasses microphone, check French dictation
|
||||
(including accents), auto-detection, English rollback, continuous capture while
|
||||
changing language, cancellation/resumption of model download, and persistence
|
||||
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
|
||||
No paid API call, microphone recording, or firmware update is needed for the
|
||||
software checks above; recognition accuracy and latency require device testing.
|
||||
|
||||
## References
|
||||
|
||||
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
||||
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
||||
|
||||
## OpenAI transcription model selection
|
||||
|
||||
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
||||
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
||||
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
||||
The unset default retains gpt-realtime-whisper for existing installations; no
|
||||
provider, language or Assistant model preference is overwritten. This setting
|
||||
applies to Translate's speech recognition as well as Transcribe and dictation.
|
||||
The two transcribe alternatives produce transcript deltas after a committed
|
||||
audio segment; existing pause segmentation and push-to-talk release supply the
|
||||
commits. Choosing a model retargets active continuous capture.
|
||||
|
||||
Model access errors name the requested model before the long project error and
|
||||
point to the setting. Factory/session-wire tests cover model, language, manual
|
||||
commit and final transcript behavior; settings tests cover persistence and the
|
||||
Android/iOS option plumbing. No real API inference was performed; actual project
|
||||
access, recognition and latency remain to be checked on a device. The model list
|
||||
provided by the operator establishes account visibility, not a successful audio
|
||||
request.
|
||||
|
||||
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/** The English-only files remain usable after upgrading; other languages need multilingual base. */
|
||||
data class WhisperModelFiles(val language: String) {
|
||||
val englishOnly: Boolean get() = language == "en"
|
||||
val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
|
||||
private val prefix: String get() = if (englishOnly) "base.en" else "base"
|
||||
val encoder: String get() = "$prefix-encoder.int8.onnx"
|
||||
val decoder: String get() = "$prefix-decoder.int8.onnx"
|
||||
val tokens: String get() = "$prefix-tokens.txt"
|
||||
val files: List<String> get() = listOf(encoder, decoder, tokens)
|
||||
|
||||
companion object {
|
||||
/** Null preserves the old English default; an empty code enables automatic detection. */
|
||||
fun forLanguage(language: String?): WhisperModelFiles =
|
||||
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
|
||||
}
|
||||
}
|
||||
-2
@@ -37,8 +37,6 @@ class ConnectionOptions {
|
||||
const val HEARTBEAT_FAILURE_DEADLINE_MS = 10_000
|
||||
const val HEARTBEAT_READY_MS = 4_000
|
||||
const val HEARTBEAT_URGENT_MS = 6_000
|
||||
// Silent-off pushes are not reliable; promptly re-read the state while silent.
|
||||
const val SILENT_MODE_POLL_MS = 2_000
|
||||
const val BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000
|
||||
const val BATTERY_INPUT_QUIET_MS = 5_000
|
||||
// Poll cadence while in charging mode; also bounds how quickly we notice
|
||||
|
||||
+2
-3
@@ -1041,9 +1041,8 @@ internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean {
|
||||
&& sessionReady
|
||||
&& pendingMessages.isEmpty()
|
||||
&& inFlightMessages.isEmpty()
|
||||
&& (silentMode == 1 || now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS)
|
||||
&& (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >=
|
||||
if (silentMode == 1) ConnectionOptions.SILENT_MODE_POLL_MS else ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS)
|
||||
&& now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS
|
||||
&& (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS)
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessage {
|
||||
|
||||
@@ -55,6 +55,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
|
||||
ASR_MODELS: {
|
||||
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
||||
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
||||
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
|
||||
},
|
||||
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
||||
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
||||
|
||||
@@ -26,6 +26,7 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
|
||||
@@ -207,6 +207,8 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
// A running capture is shared only on the same provider and language.
|
||||
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
||||
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
||||
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
|
||||
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
|
||||
const statuses = [];
|
||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||
@@ -225,13 +227,13 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
});
|
||||
|
||||
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
||||
for (const language of ['ja', '']) {
|
||||
for (const language of ['fr', 'ja', '']) {
|
||||
const elevenlabs = harness('elevenlabs', true, { language });
|
||||
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
||||
|
||||
const soniox = harness('soniox', true, { language });
|
||||
soniox.open();
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
|
||||
|
||||
const openai = harness('openai', true, { language });
|
||||
openai.open();
|
||||
@@ -240,3 +242,43 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
||||
assert.equal(transcription.language, language || undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
||||
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
||||
const h = harness('openai', true, { model, language: 'fr' });
|
||||
h.open();
|
||||
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
||||
assert.equal(input.transcription.model, model);
|
||||
assert.equal(input.transcription.language, 'fr');
|
||||
assert.equal(input.turn_detection, null);
|
||||
h.client.acceptPcm(pcm(1200, 150));
|
||||
h.client.acceptPcm(pcm(0, 1500));
|
||||
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
||||
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
||||
assert.equal(h.events.at(-1).isFinal, true);
|
||||
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
||||
assert.ok(h.errors[0].includes(model));
|
||||
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
||||
const created = [];
|
||||
const load = loader({ setTimeout, clearTimeout }, {
|
||||
'./openai-stt': {
|
||||
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
||||
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
||||
},
|
||||
});
|
||||
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
||||
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
||||
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
||||
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
||||
assert.equal(created[0].language, 'fr');
|
||||
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
||||
});
|
||||
@@ -12,6 +12,7 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
|
||||
@@ -6,7 +6,8 @@ const ts = require('typescript');
|
||||
const path = require('node:path');
|
||||
// The provider settings ios-preview-controller hands the voice bridge.
|
||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
getVoiceTranscriptionLanguage: () => '',
|
||||
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||
function load(file, context) {
|
||||
@@ -168,6 +169,7 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
|
||||
let cachedFont = 'Light';
|
||||
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
|
||||
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
||||
setTimeout: fn => tasks.push(fn),
|
||||
});
|
||||
@@ -384,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
||||
recordTo: name => { bridge.recording = name; },
|
||||
setRawSession: session => { bridge.rawSession = session; } };
|
||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
|
||||
saveTranscribeAudioSetting: { get: () => true } };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||
|
||||
@@ -316,24 +316,6 @@ private class Session {
|
||||
}
|
||||
|
||||
class GlassesSessionCoreTest {
|
||||
@Test fun silentModePollsPromptlyWithoutChangingNormalBatteryCadence() {
|
||||
val s = Session()
|
||||
val core = s.core
|
||||
core.monitor.withLock {
|
||||
core.sessionReady = true
|
||||
core.silentMode = 1
|
||||
core.lastBatteryRefreshAtMs = 8_000L
|
||||
core.lastConnectionOrInputAtMs = 10_000L
|
||||
assertFalse(core.shouldPollBatteryLocked(9_999L))
|
||||
assertTrue(core.shouldPollBatteryLocked(10_000L))
|
||||
core.silentMode = 0
|
||||
assertFalse(core.shouldPollBatteryLocked(10_000L))
|
||||
assertTrue(core.shouldPollBatteryLocked(308_000L))
|
||||
core.shutdownRequested = true
|
||||
assertFalse(core.shouldPollBatteryLocked(608_000L))
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun brightnessConfigPrecedesFirstFrameAndSleepWakeFollowsComposites() {
|
||||
val s = Session()
|
||||
s.core.configureBrightness(false, 70, 2, 100, "0:0,1000:100", 280)
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class WhisperModelFilesTest {
|
||||
@Test fun existingEnglishFilesRemainUsable() {
|
||||
for (language in listOf(null, "en", " EN ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertTrue(model.englishOnly)
|
||||
assertEquals("en", model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
|
||||
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
|
||||
for (language in listOf("fr", "de", "ja", " FR ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals(language.trim().lowercase(), model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
|
||||
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
|
||||
val model = WhisperModelFiles.forLanguage("")
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals("", model.language)
|
||||
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
function fixture() {
|
||||
const source = ts.createSourceFile('controller.ts', fs.readFileSync('app/g2/dashboard-controller.ts', 'utf8'), ts.ScriptTarget.Latest, true);
|
||||
const cls = source.statements.find(n => ts.isClassDeclaration(n) && n.name?.text === 'DashboardController');
|
||||
const methods = cls.members.filter(n => ['handleSilentMode', 'showSilentModeExit', 'setPhase'].includes(n.name?.getText(source)));
|
||||
const alerts = [], timers = new Map(); let next = 0, on = true, wakes = 0;
|
||||
const context = { exports: {}, setTimeout: fn => { timers.set(++next, fn); return next; }, clearTimeout: id => timers.delete(id),
|
||||
updateGlassesPresence() {}, shell: { isScreenOn: () => on, wake: () => { on = true; wakes++; }, showAlert: text => alerts.push(text),
|
||||
foregroundWindow: () => ({ requestRender() {} }), setBatteryLevels() {}, getBatteryLevels: () => ({}) } };
|
||||
vm.runInNewContext(ts.transpileModule(`export class Controller { ${methods.map(n => n.getText(source)).join('\n')} }`, {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, context);
|
||||
const c = new context.exports.Controller();
|
||||
Object.assign(c, { silentMode: false, silentModeFeedback: '', silentModeFeedbackTimer: null, silentModeFeedbackGeneration: 0, phase: 'connected', emit() {},
|
||||
cancelEvenHubSuspendTimer() {}, ensureEvenHubSessionActive: async () => true, requestShellRender() {},
|
||||
appendLog() {}, formatError: error => String(error) });
|
||||
return { c, alerts, timers, wakes: () => wakes, dark: () => { on = false; }, expire() { const callbacks = [...timers.values()]; timers.clear(); callbacks.forEach(fn => fn()); } };
|
||||
}
|
||||
test('silent activation remains visible on phone; exit waits for the display wake barrier', async () => {
|
||||
const h = fixture(); h.c.handleSilentMode(false);
|
||||
assert.equal(h.c.silentModeFeedback, '');
|
||||
h.c.handleSilentMode(true);
|
||||
assert.match(h.c.silentModeFeedback, /enabled/); assert.equal(h.timers.size, 0); assert.equal(h.alerts.length, 0);
|
||||
h.c.handleSilentMode(false);
|
||||
assert.equal(h.c.silentModeFeedback, 'Silent mode disabled');
|
||||
await new Promise(setImmediate);
|
||||
assert.deepEqual(h.alerts, ['Silent mode disabled']);
|
||||
h.expire(); assert.equal(h.c.silentModeFeedback, '');
|
||||
});
|
||||
test('repeated state reports do not duplicate feedback; an old timer cannot clear a new activation', async () => {
|
||||
const h = fixture(); h.c.handleSilentMode(true); h.c.handleSilentMode(false);
|
||||
h.c.handleSilentMode(false); await new Promise(setImmediate); assert.equal(h.alerts.length, 1);
|
||||
h.c.handleSilentMode(true); h.expire();
|
||||
assert.match(h.c.silentModeFeedback, /enabled/);
|
||||
});
|
||||
test('silent exit wakes a sleeping display before showing feedback but leaves locked glasses alone', async () => {
|
||||
for (const locked of [false, true]) {
|
||||
const h = fixture(); h.c.glassesLocked = locked; h.dark();
|
||||
h.c.handleSilentMode(true); h.c.handleSilentMode(false); await new Promise(setImmediate);
|
||||
assert.equal(h.wakes(), locked ? 0 : 1);
|
||||
assert.equal(h.alerts.length, locked ? 0 : 1);
|
||||
assert.equal(h.c.silentModeFeedback, 'Silent mode disabled');
|
||||
}
|
||||
});
|
||||
test('an unavailable session, re-entry into silence or disconnect suppresses a stale exit popup', async () => {
|
||||
for (const state of ['not-ready', 'silent', 'disconnected']) {
|
||||
const h = fixture(); let finish;
|
||||
h.c.ensureEvenHubSessionActive = () => new Promise(resolve => { finish = resolve; });
|
||||
h.c.handleSilentMode(true); h.c.handleSilentMode(false);
|
||||
assert.equal(h.alerts.length, 0);
|
||||
if (state === 'silent') h.c.handleSilentMode(true);
|
||||
if (state === 'disconnected') h.c.setPhase('disconnected');
|
||||
finish(state !== 'not-ready'); await new Promise(setImmediate);
|
||||
assert.equal(h.alerts.length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('disconnect clears feedback and any pending timer without announcing a silent exit', () => {
|
||||
const h = fixture(); h.c.handleSilentMode(true); h.c.handleSilentMode(false);
|
||||
h.c.setPhase('disconnected');
|
||||
assert.equal(h.c.silentModeFeedback, ''); assert.equal(h.timers.size, 0);
|
||||
h.expire(); assert.equal(h.c.silentModeFeedback, '');
|
||||
});
|
||||
@@ -28,6 +28,7 @@
|
||||
"../app/ui/menu-core.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/native/java-direct-buffer.ts",
|
||||
"../app/native/transcription-language.ts",
|
||||
"../app/native/notification-types.ts",
|
||||
"../app/g2/firmware-compat.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
|
||||
@@ -163,7 +163,8 @@ function androidBridge() {
|
||||
starts: 0,
|
||||
setListener(listener) { this.listener = listener; },
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
|
||||
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
|
||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||
stop() { this.active = false; },
|
||||
isCapturing() { return this.active; },
|
||||
@@ -171,7 +172,8 @@ function androidBridge() {
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null,
|
||||
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: function () { return controller; },
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
@@ -179,6 +181,26 @@ function androidBridge() {
|
||||
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
||||
}
|
||||
|
||||
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.modelKind, 'whisper');
|
||||
assert.equal(controller.language, 'fr');
|
||||
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.starts, 1);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
|
||||
assert.equal(controller.language, '');
|
||||
assert.equal(controller.starts, 2);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
||||
// An omitted language preserves the native bridge's original English behavior.
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.stop();
|
||||
});
|
||||
|
||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
const options = { provider: 'onboard-whisper' };
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, requireModule = () => ({}), globals = {}) {
|
||||
const exports = {};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
|
||||
return exports;
|
||||
}
|
||||
const languages = load('app/native/transcription-language.ts');
|
||||
function settings(values = new Map()) {
|
||||
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
|
||||
: name.includes('settings-store') ? {
|
||||
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
||||
setStringSetting: (key, value) => values.set(key, value),
|
||||
getBooleanSetting: (_key, fallback) => fallback,
|
||||
onSettingsStoreChanged() {},
|
||||
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
|
||||
}
|
||||
|
||||
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
const s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
|
||||
});
|
||||
|
||||
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
let s = settings(values);
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
|
||||
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
|
||||
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
|
||||
}
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
|
||||
s.voiceLanguageSetting.set('auto');
|
||||
s = settings(values);
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
s.voiceLanguageSetting.set('default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
});
|
||||
|
||||
test('corrupt language preferences revert to the compatible provider defaults', () => {
|
||||
const s = settings(new Map([['voice.language', 'invalid']]));
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
});
|
||||
|
||||
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
|
||||
const s = settings();
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s.voiceProviderSetting.set('whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
for (const [file, name, method] of [
|
||||
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
||||
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
||||
]) {
|
||||
const host = { continuousClients: new Map() };
|
||||
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
|
||||
// Run each real capture-options method with only its native dependencies stubbed.
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
|
||||
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
||||
const capture = loadBody(member.body.getText(tree), s, host);
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
||||
assert.equal(capture().provider, 'soniox');
|
||||
assert.equal(capture().language, 'ja');
|
||||
host.continuousClients.clear();
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
}
|
||||
});
|
||||
|
||||
function loadBody(body, s, host) {
|
||||
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
|
||||
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
|
||||
}
|
||||
|
||||
test('multilingual downloads and deletion leave an existing English model intact', () => {
|
||||
const files = new Map(), downloads = [];
|
||||
const asr = load('app/native/asr-model.ts', () => ({
|
||||
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
||||
}), {
|
||||
java: { io: { File: class {
|
||||
constructor(name) { this.name = name; }
|
||||
exists() { return files.has(this.name); }
|
||||
length() { return files.get(this.name) ?? 0; }
|
||||
delete() { return files.delete(this.name); }
|
||||
} } },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
||||
FaceclawModelDownloader: class {
|
||||
constructor(url, destination, hash, size, listener) {
|
||||
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
||||
}
|
||||
start() {}
|
||||
cancel() {}
|
||||
},
|
||||
} } },
|
||||
});
|
||||
const english = asr.ASR_MODELS['whisper-base-en'];
|
||||
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
asr.startAsrModelDownload('whisper-base');
|
||||
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
|
||||
for (let index = 0; index < 3; index++) {
|
||||
const download = downloads[index];
|
||||
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
|
||||
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
||||
files.set(download.destination, download.size);
|
||||
download.listener.onDone();
|
||||
}
|
||||
assert.equal(downloads.length, 3);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), true);
|
||||
asr.deleteAsrModel('whisper-base');
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
});
|
||||
|
||||
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
||||
const values = new Map();
|
||||
let s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
||||
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
});
|
||||
Reference in new issue
Block a user