forked from Blendman974/faceclaw
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe87e75fc2
|
||
|
|
f029e3ce91
|
||
|
|
5929259d84
|
No files matched your search
@@ -25,21 +25,18 @@ internal object AndroidSpeechEngines {
|
||||
private const val FEATURE_DIM = 80
|
||||
|
||||
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
||||
fun recognizerConfig(
|
||||
modelDir: File, kind: VoiceModelKind,
|
||||
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
|
||||
): OfflineRecognizerConfig {
|
||||
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
|
||||
val modelConfig = OfflineModelConfig.builder()
|
||||
.setNumThreads(1)
|
||||
if (kind == VoiceModelKind.WHISPER) {
|
||||
modelConfig
|
||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
|
||||
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
|
||||
.setLanguage(whisper.language)
|
||||
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
|
||||
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
|
||||
.setLanguage("en")
|
||||
.setTask("transcribe")
|
||||
.build())
|
||||
.setTokens(File(modelDir, whisper.tokens).absolutePath)
|
||||
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
|
||||
} else {
|
||||
modelConfig
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
|
||||
@@ -249,6 +249,8 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun enableWearDetectionAndRequestState() = core.enableWearDetectionAndRequestState()
|
||||
|
||||
fun configureHeadUpWake(enabled: Int, angle: Int) = core.configureHeadUpWake(enabled, angle)
|
||||
|
||||
fun startBandwidthBenchmark(messageSize: Int, windowSize: Int, durationMs: Int): Boolean =
|
||||
core.startBandwidthBenchmark(messageSize, windowSize, durationMs)
|
||||
|
||||
|
||||
@@ -32,6 +32,17 @@ class FaceclawVoiceController(context: Context) {
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
)
|
||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||
// Whisper re-encodes the whole buffer on every call, so the shared session
|
||||
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
|
||||
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
|
||||
private val ASR_WHISPER_MODEL_FILES = arrayOf(
|
||||
"base.en-encoder.int8.onnx",
|
||||
"base.en-decoder.int8.onnx",
|
||||
"base.en-tokens.txt"
|
||||
)
|
||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||
// (endpointing, transcript pacing) is tuned for.
|
||||
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
||||
@@ -73,7 +84,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
@Volatile
|
||||
private var communicator: FaceclawBleCommunicator? = null
|
||||
private var whisperModel = WhisperModelFiles.forLanguage(null)
|
||||
|
||||
private val host = object : VoiceCaptureHost {
|
||||
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
||||
@@ -112,7 +122,7 @@ class FaceclawVoiceController(context: Context) {
|
||||
|
||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
||||
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
|
||||
}
|
||||
|
||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
||||
@@ -158,11 +168,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setOnboardModelKind(kind)
|
||||
}
|
||||
|
||||
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
|
||||
fun setOnboardLanguage(language: String?) {
|
||||
whisperModel = WhisperModelFiles.forLanguage(language)
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
@@ -258,8 +263,8 @@ class FaceclawVoiceController(context: Context) {
|
||||
*/
|
||||
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
||||
val whisper = kind == VoiceModelKind.WHISPER
|
||||
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
|
||||
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
|
||||
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
||||
for (fileName in fileNames) {
|
||||
val file = File(modelDir, fileName)
|
||||
|
||||
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -296,6 +296,7 @@ class DashboardController {
|
||||
private silentMode = false;
|
||||
// Brightness value last pushed to the glasses; see pushBrightness.
|
||||
private lastPushedBrightness: string | null = null;
|
||||
private lastPushedHeadUp: string | null = null;
|
||||
// Last battery level the glasses reported, kept across disconnects so a
|
||||
// drop-off right after a low reading can be explained as a flat battery.
|
||||
private lastHeadsetBattery: number | null = null;
|
||||
@@ -476,6 +477,7 @@ class DashboardController {
|
||||
// Apply a firmware-debug-flags toggle live while connected (Java dedups).
|
||||
this.pushFirmwareDebugFlags();
|
||||
this.pushBrightness();
|
||||
this.pushHeadUpPreferences();
|
||||
this.syncEvenHubScreenOffSetting();
|
||||
// Display mode size first: a reflow repositions every surface too, so
|
||||
// the band-position check after it then has nothing left to do.
|
||||
@@ -483,7 +485,6 @@ class DashboardController {
|
||||
this.applyVerticalPositionIfChanged();
|
||||
this.applyAppLayoutsIfChanged();
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
this.retargetContinuousCapture();
|
||||
this.syncLockScreenSettingIfChanged();
|
||||
// A Preview color or scaling change should show on the mirror at once,
|
||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||
@@ -1085,6 +1086,19 @@ class DashboardController {
|
||||
});
|
||||
}
|
||||
|
||||
private pushHeadUpPreferences(force = false): void {
|
||||
if (!global.isAndroid || !this.communicator) return;
|
||||
const { enabled, angle } = getHeadUpPreferences();
|
||||
if (enabled < 0 && angle < 0) return;
|
||||
const value = JSON.stringify({ enabled, angle });
|
||||
if (!force && value === this.lastPushedHeadUp) return;
|
||||
this.lastPushedHeadUp = value;
|
||||
void this.communicator.configureHeadUpWake(enabled, angle).catch(error => {
|
||||
this.lastPushedHeadUp = null;
|
||||
this.appendLog(`head-up configuration failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
/** Save the occupied part of the composited screen as a 4-bit grayscale PNG. */
|
||||
saveScreenshot(): string {
|
||||
const path = this.display?.saveScreenshot(shell.screenshotCropRect()) ?? "";
|
||||
@@ -1534,6 +1548,7 @@ class DashboardController {
|
||||
// ahead of them. Mirrors ensurePreviewDisplay.
|
||||
await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
|
||||
await communicator.configureBrightness(getBrightnessPreferences());
|
||||
this.pushHeadUpPreferences(true);
|
||||
this.offState = communicator.onStateChange((state) => {
|
||||
if (state.phase !== "connected") resetRingInputFilter();
|
||||
if (isHaltedSessionPhase(state.phase)) {
|
||||
@@ -1571,6 +1586,7 @@ class DashboardController {
|
||||
// mode-7 control message once the dashboard container is warmed up.
|
||||
this.pushFirmwareDebugFlags();
|
||||
this.pushBrightness(true);
|
||||
this.pushHeadUpPreferences(true);
|
||||
this.handRingBackToGlassesIfNeeded();
|
||||
}
|
||||
if (mappedPhase !== "connected" && mappedPhase !== "charging") {
|
||||
@@ -2198,16 +2214,16 @@ class DashboardController {
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise use the shared transcription-language preference.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider,
|
||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
|
||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
@@ -416,7 +416,6 @@ export class IosPreviewController {
|
||||
const phone = readPhoneBatteryState()
|
||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.syncContinuousCapture()
|
||||
this.syncLockSetting()
|
||||
this.pushBrightness()
|
||||
this.relayout()
|
||||
@@ -1199,13 +1198,14 @@ export class IosPreviewController {
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise use the shared language preference.
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
|
||||
+2
-32
@@ -9,7 +9,7 @@ declare const java: any;
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Downloadable models, same shape:
|
||||
* Two models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
@@ -29,17 +29,12 @@ declare const java: any;
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
|
||||
* speech and auto-detection; English-only downloads remain untouched.
|
||||
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
||||
* vocabulary hash computed locally. URLs pin the source revision.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base";
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
@@ -110,30 +105,6 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
|
||||
"whisper-base": {
|
||||
label: "Whisper (multilingual, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-int8",
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
|
||||
files: [
|
||||
{
|
||||
name: "base-encoder.int8.onnx",
|
||||
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base-decoder.int8.onnx",
|
||||
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
|
||||
sizeBytes: 130672026,
|
||||
},
|
||||
{
|
||||
name: "base-tokens.txt",
|
||||
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
||||
sizeBytes: 816730,
|
||||
},
|
||||
],
|
||||
totalBytes: 160609290,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -158,7 +129,6 @@ function freshRuntime(): ModelRuntime {
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
"whisper-base": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
@@ -14,12 +14,11 @@ export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Also used by on-device Whisper.
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
@@ -53,7 +52,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
whisper: {
|
||||
name: "OpenAI",
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
},
|
||||
"whisper-server": {
|
||||
name: "Whisper server",
|
||||
@@ -77,10 +76,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
||||
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
|
||||
@@ -519,6 +519,11 @@ export class FaceclawCommunicatorBridge {
|
||||
await this.enqueueJavaCall(() => this.communicator.enableWearDetectionAndRequestState());
|
||||
}
|
||||
|
||||
/** -1 leaves a field unchanged. The native session retains explicit values across reconnects. */
|
||||
async configureHeadUpWake(enabled: number, angle: number): Promise<void> {
|
||||
await this.enqueueJavaCall(() => this.communicator.configureHeadUpWake(enabled, angle));
|
||||
}
|
||||
|
||||
/** Whether the transport prelude has finished (each arm reports "connected" before it). */
|
||||
isSessionReady(): boolean {
|
||||
return Boolean(this.communicator.isSessionReady());
|
||||
|
||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
* on the Realtime API, with a configurable transcription model).
|
||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||
*
|
||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||
@@ -16,11 +16,10 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
||||
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
||||
const MODEL_ID = "gpt-realtime-whisper";
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
|
||||
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: SocketConnection | null = null;
|
||||
@@ -41,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -166,9 +165,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(message.error?.code === "model_not_found"
|
||||
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
||||
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
@@ -204,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
||||
function sessionConfig(language: string): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -212,7 +209,7 @@ function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model, ...(language ? { language } : {}) },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
/** Languages supported by both Whisper backends, in picker order. */
|
||||
export const TRANSCRIPTION_LANGUAGES = {
|
||||
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
|
||||
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
|
||||
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
|
||||
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
|
||||
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
|
||||
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
|
||||
|
||||
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
|
||||
export function resolveTranscriptionLanguage(
|
||||
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
||||
): string {
|
||||
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
||||
if (provider === "onboard") return "";
|
||||
if (choice === "default") {
|
||||
if (provider === "onboard-whisper") return "en";
|
||||
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
|
||||
}
|
||||
return choice === "auto" ? "" : choice;
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { OpenAiSttModel } from "./openai-stt";
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
@@ -25,7 +24,7 @@ export type VoiceControlState = {
|
||||
detail: string;
|
||||
};
|
||||
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
@@ -55,12 +54,11 @@ export type PushToTalkOptions = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
@@ -398,7 +396,6 @@ export class FaceclawVoiceControlBridge {
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
this.controller?.setOnboardLanguage(options.language ?? "en");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { OpenAiSttModel } from "../native/openai-stt";
|
||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||
@@ -19,7 +18,6 @@ import {
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
|
||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -557,6 +555,38 @@ export const brightnessSetting = new ConfigSettingEnum<BrightnessSetting>({
|
||||
description: "Auto uses Faceclaw's ambient-light curve; numbers set a fixed level. 2 is the dimmest level.",
|
||||
});
|
||||
|
||||
/** String storage distinguishes an untouched firmware preference from explicit ON/OFF. */
|
||||
class HeadUpWakeSetting extends ConfigSettingBoolean {
|
||||
get(): boolean { return getStringSetting(this.storageKey, "true") !== "false"; }
|
||||
set(value: boolean): boolean { setStringSetting(this.storageKey, String(value)); return value; }
|
||||
}
|
||||
|
||||
export const headUpWakeSetting = new HeadUpWakeSetting({
|
||||
id: "head-up-wake", label: "Head-up wake", storageKey: "display.headUpWake", defaultValue: true,
|
||||
description: "Enable waking the glasses when you look up. Until changed, the glasses keep their current setting. Glanceboard's Show on head tilt chooses what appears after waking.",
|
||||
});
|
||||
|
||||
export function normalizeHeadUpAngle(value: string | null | undefined): string {
|
||||
if (value == null || value.trim() === "" || !Number.isFinite(Number(value))) return "30";
|
||||
return String(Math.max(0, Math.min(60, Math.round(Number(value)))));
|
||||
}
|
||||
|
||||
export const headUpAngleSetting = new ConfigSettingEnum({
|
||||
id: "head-up-angle", label: "Head-up angle", storageKey: "display.headUpAngle", defaultValue: "30",
|
||||
values: Array.from({ length: 61 }, (_, angle) => String(angle)),
|
||||
normalize: normalizeHeadUpAngle,
|
||||
formatValue: angle => `${angle}°`,
|
||||
description: "Look-up trigger angle, 0–60 degrees. Until changed, the glasses keep their current angle. Requires validation on your Faceclaw firmware.",
|
||||
});
|
||||
|
||||
/** -1 means leave that firmware field unchanged; no writes on an untouched installation. */
|
||||
export function getHeadUpPreferences(): { enabled: number; angle: number } {
|
||||
const enabled = getStringSetting("display.headUpWake", "");
|
||||
const angle = getStringSetting("display.headUpAngle", "");
|
||||
return { enabled: enabled === "true" ? 1 : enabled === "false" ? 0 : -1,
|
||||
angle: angle === "" ? -1 : Number(normalizeHeadUpAngle(angle)) };
|
||||
}
|
||||
|
||||
const AUTO_BRIGHTNESS_LEVELS = ["2", "5", "10", "20", "25", "30", "40", "50", "60", "70", "80", "90", "100"] as const;
|
||||
export const autoBrightnessMinSetting = new ConfigSettingEnum({
|
||||
id: "autoBrightnessMin", label: "Auto minimum", storageKey: "display.autoBrightnessMin",
|
||||
@@ -1138,35 +1168,9 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||
});
|
||||
|
||||
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
||||
id: "openai-transcription-model",
|
||||
label: "OpenAI transcription model",
|
||||
storageKey: "voice.openAiTranscriptionModel",
|
||||
defaultValue: "gpt-realtime-whisper",
|
||||
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
||||
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
||||
});
|
||||
|
||||
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
||||
id: "voice-language",
|
||||
label: "Transcription language",
|
||||
storageKey: "voice.language",
|
||||
defaultValue: "default",
|
||||
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
|
||||
formatValue: (value) => value === "default" ? "Provider default"
|
||||
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
||||
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
||||
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.",
|
||||
});
|
||||
|
||||
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
||||
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
|
||||
}
|
||||
|
||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
id: "whisper-server-language",
|
||||
label: "Whisper server language",
|
||||
label: "Whisper language",
|
||||
storageKey: "voice.whisperServerLanguage",
|
||||
defaultValue: "",
|
||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||
@@ -1174,7 +1178,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
formatValue: (value) => value || "Auto-detect",
|
||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||
description:
|
||||
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
|
||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||
});
|
||||
|
||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -49,6 +49,8 @@ import {
|
||||
veniceWebSearchSetting,
|
||||
batteryDisplayModeSetting,
|
||||
brightnessSetting,
|
||||
headUpWakeSetting,
|
||||
headUpAngleSetting,
|
||||
autoBrightnessMinSetting,
|
||||
autoBrightnessMaxSetting,
|
||||
autoBrightnessCurveSetting,
|
||||
@@ -63,7 +65,6 @@ import {
|
||||
mapboxApiKeySetting,
|
||||
mirrorTouchSetting,
|
||||
openAiApiKeySetting,
|
||||
openAiTranscriptionModelSetting,
|
||||
previewColorSetting,
|
||||
previewScalingSetting,
|
||||
phoneRotationSetting,
|
||||
@@ -88,7 +89,6 @@ import {
|
||||
toggleSettingMenuItem,
|
||||
useMicControlSetting,
|
||||
voiceProviderSetting,
|
||||
voiceLanguageSetting,
|
||||
screenTimeoutSetting,
|
||||
wakeToLastAppSetting,
|
||||
wakeWordActionSetting,
|
||||
@@ -120,6 +120,7 @@ function settingsSections(): SettingsSection[] {
|
||||
// the dashboard controller when changed and on each connect.
|
||||
enumSettingMenuItem(brightnessSetting),
|
||||
autoBrightnessMenuItem(),
|
||||
...(!global.isIOS ? [toggleSettingMenuItem(headUpWakeSetting), enumSettingMenuItem(headUpAngleSetting)] : []),
|
||||
enumSettingMenuItem(screenTimeoutSetting, {
|
||||
onChange: () => {
|
||||
shell.noteUserActivity();
|
||||
@@ -145,11 +146,9 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
enumSettingMenuItem(voiceLanguageSetting),
|
||||
...whisperServerMenuItems(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
asrModelMenuItem("whisper-base"),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -288,7 +287,7 @@ function settingsSections(): SettingsSection[] {
|
||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||
item.label !== screenTimeoutSetting.label) };
|
||||
return section;
|
||||
@@ -531,7 +530,6 @@ function asrModelStatusText(id: AsrModelId): string {
|
||||
*/
|
||||
function whisperServerMenuItems(): MenuItem[] {
|
||||
return [
|
||||
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
||||
textSettingMenuItem(whisperServerUrlSetting),
|
||||
textSettingMenuItem(whisperServerModelSetting),
|
||||
textSettingMenuItem(whisperServerLanguageSetting),
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Head-up wake: software and hardware validation
|
||||
|
||||
The Android Display settings now contain “Head-up wake” and “Head-up angle”.
|
||||
Glanceboard's “Show on head tilt” still only chooses which UI appears on wake.
|
||||
|
||||
Protocol reference: [OpenG2 Companion G2Protocol.swift](https://github.com/mmirman/open-g2-companion/blob/1400d1006ba045379a3ff34d0832add6fa10ced4/modules/ffs-ble/ios/G2Protocol.swift),
|
||||
`setHeadUpSwitch` / `setHeadUpAngle`; dispatch to both lenses in
|
||||
[`G2Central.swift`](https://github.com/mmirman/open-g2-companion/blob/1400d1006ba045379a3ff34d0832add6fa10ced4/modules/ffs-ble/ios/G2Central.swift).
|
||||
|
||||
Wire format uses Faceclaw's UI-settings sid 0x09, request flag, command 1, transaction
|
||||
magic, device-receive-info field 3 / head-up field 4. Switch (field 1) and
|
||||
angle (field 2) are sent in separate messages, including explicit zero values.
|
||||
Angles are clamped to the reference's 0–60 degree range. No firmware patch,
|
||||
firmware download or flash is needed by this implementation.
|
||||
|
||||
Untouched installations send neither field. The UI describes this inherited
|
||||
state; its displayed ON/30 degree fallback is not a read-back of device state.
|
||||
Changing just one control preserves the other device field. Explicit choices
|
||||
are persisted in the existing settings store and resent on every connection.
|
||||
The native session also retains them through transport reconnects and retries
|
||||
after setting ACK timeouts. ACK reception confirms transport, not physical wake.
|
||||
|
||||
Software checks:
|
||||
|
||||
- `tests/head-up-wake.test.cjs`: persistence, untouched fields, angle bounds,
|
||||
independence from Glanceboard, deduplication, reconnect and error recovery.
|
||||
- `HeadUpProtocolTest`: byte-for-byte settings messages, including OFF and 0°.
|
||||
- `GlassesSessionCoreTest`: both lenses, separate writes, pre-existing device
|
||||
settings preserved, angle clamp and automatic transport reconnection.
|
||||
|
||||
Hardware validation: **pending**. Implementation was requested without device
|
||||
access; protocol compatibility and physical effect on the target Faceclaw CFW
|
||||
are not established by these software tests. The reference targets stock
|
||||
2.2.6.10, while this repository normally uses newer Faceclaw firmware.
|
||||
|
||||
Source inspection of [Faceclaw's gesture forwarding](https://github.com/jimrandomh/g2flash/blob/845f665e0dfdcfd06b7d3a294388de6f17a5bd59/patches/gesture_fwd.c)
|
||||
confirms that head-up/down forwarding while a page owns the screen checks the
|
||||
stock `head_up_switch`. This supports using the existing setting rather than a
|
||||
firmware patch, but does not establish the physical angle or wake behavior.
|
||||
|
||||
On the target glasses, record both lens firmware versions and test:
|
||||
|
||||
1. Leave preferences untouched and verify prior wake behavior remains unchanged.
|
||||
2. Disable Head-up wake; sleep the display, raise the head repeatedly, and confirm
|
||||
no wake. Check taps still wake normally.
|
||||
3. Enable it and compare 0°, 30° and 60° trigger angles. Confirm other wake
|
||||
gestures remain usable, including at the extreme values.
|
||||
4. Reconnect Bluetooth, then restart the phone app; repeat ON/OFF and angle checks.
|
||||
5. Toggle Glanceboard and “Show on head tilt” independently; disabled wake should
|
||||
stay disabled, enabled wake should show the selected UI.
|
||||
6. Check both lenses remain synchronized, including reconnecting only one arm.
|
||||
|
||||
Do not change firmware to run these checks without explicit permission.
|
||||
@@ -1,68 +0,0 @@
|
||||
# Whisper transcription languages
|
||||
|
||||
Settings > Voice > Transcription language selects French, English, 27 other
|
||||
languages, or auto-detection. It applies to ordinary dictation and continuous
|
||||
transcription using OpenAI, on-device Whisper, and the other cloud providers.
|
||||
Translate retains its own spoken-language preference. Moonshine remains
|
||||
English-only; Apple's on-device speech recognizer retains its own locale.
|
||||
|
||||
The initial **Provider default** preserves existing installations: English for
|
||||
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
|
||||
Whisper server language for self-hosted Whisper. Selecting another shared
|
||||
language overrides the server language without deleting it. Returning to Provider
|
||||
default restores it. Settings persist through the existing settings store.
|
||||
|
||||
For French or auto-detection with **On-device (Whisper)**, download
|
||||
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
|
||||
English uses the existing English-only base model; neither model is deleted or
|
||||
downloaded automatically. A missing model uses the existing “Voice model not
|
||||
downloaded” status. Language changes restart active continuous transcription;
|
||||
other captures use the new language when started.
|
||||
|
||||
## Software validation
|
||||
|
||||
- JavaScript regression tests exercise stored settings/reload, compatible
|
||||
defaults, French and auto-detection, Translate overrides in Android/iOS capture
|
||||
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
|
||||
- Kotlin tests exercise English/French/other/automatic model selection and file
|
||||
names. Android release compilation checks the actual sherpa-onnx bridge.
|
||||
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
|
||||
the existing resumable downloader. Encoder/decoder hashes and sizes come from
|
||||
repository LFS metadata; the vocabulary hash was computed from its contents.
|
||||
|
||||
## Device validation still required
|
||||
|
||||
With both phone microphone preview and glasses microphone, check French dictation
|
||||
(including accents), auto-detection, English rollback, continuous capture while
|
||||
changing language, cancellation/resumption of model download, and persistence
|
||||
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
|
||||
No paid API call, microphone recording, or firmware update is needed for the
|
||||
software checks above; recognition accuracy and latency require device testing.
|
||||
|
||||
## References
|
||||
|
||||
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
||||
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
||||
|
||||
## OpenAI transcription model selection
|
||||
|
||||
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
||||
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
||||
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
||||
The unset default retains gpt-realtime-whisper for existing installations; no
|
||||
provider, language or Assistant model preference is overwritten. This setting
|
||||
applies to Translate's speech recognition as well as Transcribe and dictation.
|
||||
The two transcribe alternatives produce transcript deltas after a committed
|
||||
audio segment; existing pause segmentation and push-to-talk release supply the
|
||||
commits. Choosing a model retargets active continuous capture.
|
||||
|
||||
Model access errors name the requested model before the long project error and
|
||||
point to the setting. Factory/session-wire tests cover model, language, manual
|
||||
commit and final transcript behavior; settings tests cover persistence and the
|
||||
Android/iOS option plumbing. No real API inference was performed; actual project
|
||||
access, recognition and latency remain to be checked on a device. The model list
|
||||
provided by the operator establishes account visibility, not a successful audio
|
||||
request.
|
||||
|
||||
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/** The English-only files remain usable after upgrading; other languages need multilingual base. */
|
||||
data class WhisperModelFiles(val language: String) {
|
||||
val englishOnly: Boolean get() = language == "en"
|
||||
val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
|
||||
private val prefix: String get() = if (englishOnly) "base.en" else "base"
|
||||
val encoder: String get() = "$prefix-encoder.int8.onnx"
|
||||
val decoder: String get() = "$prefix-decoder.int8.onnx"
|
||||
val tokens: String get() = "$prefix-tokens.txt"
|
||||
val files: List<String> get() = listOf(encoder, decoder, tokens)
|
||||
|
||||
companion object {
|
||||
/** Null preserves the old English default; an empty code enables automatic detection. */
|
||||
fun forLanguage(language: String?): WhisperModelFiles =
|
||||
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
|
||||
}
|
||||
}
|
||||
@@ -497,6 +497,23 @@ class BleProtocol {
|
||||
)
|
||||
}
|
||||
|
||||
/** Stock HeadUpSetting: switch and angle are separate oneof writes (field 4 of device info). */
|
||||
@JvmStatic
|
||||
fun buildSetHeadUpWake(magic: Int, enabled: Boolean): ByteArray = concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1), encodeVarintField(2, magic),
|
||||
encodeMessageField(3, encodeMessageField(4, encodeVarintField(1, if (enabled) 1 else 0))),
|
||||
)
|
||||
)
|
||||
|
||||
@JvmStatic
|
||||
fun buildSetHeadUpAngle(magic: Int, angle: Int): ByteArray = concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1), encodeVarintField(2, magic),
|
||||
encodeMessageField(3, encodeMessageField(4, encodeVarintField(2, angle.coerceIn(0, 60)))),
|
||||
)
|
||||
)
|
||||
|
||||
/** Enable the firmware's wear detector (DeviceReceiveInfoFromAPP field 5). */
|
||||
@JvmStatic
|
||||
fun buildSetWearDetection(magic: Int, enabled: Boolean): ByteArray {
|
||||
|
||||
+11
@@ -188,6 +188,17 @@ class MessageBuilder {
|
||||
)
|
||||
}
|
||||
|
||||
fun headUpControl(leftArm: Boolean, enabled: Boolean?, angle: Int): OutboundMessage {
|
||||
val magic = magicPool.allocate()
|
||||
return OutboundMessage(
|
||||
"head-up-control", if (enabled == null) "head-up angle=$angle" else "head-up wake=$enabled",
|
||||
BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, magic,
|
||||
if (enabled == null) BleProtocol.buildSetHeadUpAngle(magic, angle)
|
||||
else BleProtocol.buildSetHeadUpWake(magic, enabled),
|
||||
ACK_TIMEOUT_MS, -1, leftArm,
|
||||
)
|
||||
}
|
||||
|
||||
fun createLayout(): OutboundMessage {
|
||||
var magic: Int = magicPool.allocate()
|
||||
return OutboundMessage(
|
||||
|
||||
+12
@@ -304,6 +304,8 @@ class GlassesSessionCore(
|
||||
internal var brightnessSentLevel = -1
|
||||
internal var brightnessAlsStartedAt = -2000L
|
||||
internal var brightnessDemoPolling = false
|
||||
internal var headUpPreferences: Pair<Int, Int>? = null
|
||||
internal var headUpSent: Pair<Int, Int>? = null
|
||||
internal var cfwCleanupDelivered = false
|
||||
internal var lastCfwCleanupAckMagic = 0
|
||||
|
||||
@@ -920,6 +922,15 @@ class GlassesSessionCore(
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/** Explicit preferences only; -1 leaves the corresponding firmware field alone. */
|
||||
fun configureHeadUpWake(enabled: Int, angle: Int) {
|
||||
require(enabled in -1..1)
|
||||
monitor.withLock {
|
||||
headUpPreferences = Pair(enabled, if (angle < 0) -1 else angle.coerceIn(0, 60))
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable the stock wear detector, then ask CFW to emit its current cached
|
||||
* state. The queue order matters: the query must run after the setting is
|
||||
@@ -2232,6 +2243,7 @@ class GlassesSessionCore(
|
||||
shutdownRequested = false
|
||||
fixedLayoutCreated = false
|
||||
clearAllMessagesLocked("session ready")
|
||||
headUpSent = null
|
||||
displayedFingerprint = ""
|
||||
lastAckAtMs = now()
|
||||
lastIncomingAtMs = lastAckAtMs
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/** Both arms receive separate stock settings messages; no CFW changes required. */
|
||||
internal fun GlassesSessionCore.enqueueHeadUpPreferencesLocked() {
|
||||
val desired = headUpPreferences ?: return
|
||||
if (!sessionReady || shutdownRequested || chargingMode || headUpSent == desired
|
||||
|| hasPendingOrInflightKindLocked("head-up-control")) return
|
||||
for (leftArm in listOf(true, false)) {
|
||||
// Set the angle before enabling wake. Omitted fields preserve device preferences.
|
||||
if (desired.second >= 0) {
|
||||
val message = messageBuilder.headUpControl(leftArm, null, desired.second)
|
||||
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up angle ack timeout") }
|
||||
pendingMessages.addLast(message)
|
||||
}
|
||||
if (desired.first >= 0) {
|
||||
val message = messageBuilder.headUpControl(leftArm, desired.first == 1, -1)
|
||||
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up wake ack timeout") }
|
||||
pendingMessages.addLast(message)
|
||||
}
|
||||
}
|
||||
headUpSent = desired
|
||||
}
|
||||
+1
@@ -74,6 +74,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
clearPendingMessagesLocked("after CFW cleanup")
|
||||
return 250
|
||||
}
|
||||
enqueueHeadUpPreferencesLocked()
|
||||
if (sessionReady
|
||||
&& now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS
|
||||
&& !hasPendingOrInflightKindLocked("framebuffer-lease-control")) {
|
||||
|
||||
@@ -55,7 +55,6 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
|
||||
ASR_MODELS: {
|
||||
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
||||
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
||||
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
|
||||
},
|
||||
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
||||
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
||||
|
||||
@@ -26,7 +26,6 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
|
||||
@@ -207,8 +207,6 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
// A running capture is shared only on the same provider and language.
|
||||
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
||||
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
||||
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
|
||||
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
|
||||
const statuses = [];
|
||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||
@@ -227,13 +225,13 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
});
|
||||
|
||||
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
||||
for (const language of ['fr', 'ja', '']) {
|
||||
for (const language of ['ja', '']) {
|
||||
const elevenlabs = harness('elevenlabs', true, { language });
|
||||
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
||||
|
||||
const soniox = harness('soniox', true, { language });
|
||||
soniox.open();
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
|
||||
|
||||
const openai = harness('openai', true, { language });
|
||||
openai.open();
|
||||
@@ -242,43 +240,3 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
||||
assert.equal(transcription.language, language || undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
||||
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
||||
const h = harness('openai', true, { model, language: 'fr' });
|
||||
h.open();
|
||||
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
||||
assert.equal(input.transcription.model, model);
|
||||
assert.equal(input.transcription.language, 'fr');
|
||||
assert.equal(input.turn_detection, null);
|
||||
h.client.acceptPcm(pcm(1200, 150));
|
||||
h.client.acceptPcm(pcm(0, 1500));
|
||||
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
||||
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
||||
assert.equal(h.events.at(-1).isFinal, true);
|
||||
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
||||
assert.ok(h.errors[0].includes(model));
|
||||
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
||||
const created = [];
|
||||
const load = loader({ setTimeout, clearTimeout }, {
|
||||
'./openai-stt': {
|
||||
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
||||
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
||||
},
|
||||
});
|
||||
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
||||
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
||||
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
||||
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
||||
assert.equal(created[0].language, 'fr');
|
||||
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
||||
});
|
||||
@@ -12,7 +12,6 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
const source = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
function evaluate(code, globals) {
|
||||
const context = { exports: {}, ...globals };
|
||||
vm.runInNewContext(ts.transpileModule(code, {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, context);
|
||||
return context.exports;
|
||||
}
|
||||
function settings(values = new Map()) {
|
||||
const file = ts.createSourceFile('settings.ts', source('app/ui/dashboard-settings.ts'), ts.ScriptTarget.Latest, true);
|
||||
const names = new Set(['ConfigSetting', 'ConfigSettingBoolean', 'ConfigSettingEnum', 'HeadUpWakeSetting',
|
||||
'headUpWakeSetting', 'normalizeHeadUpAngle', 'headUpAngleSetting', 'getHeadUpPreferences']);
|
||||
const code = file.statements.filter(node => names.has(node.name?.text) ||
|
||||
(ts.isVariableStatement(node) && node.declarationList.declarations.some(d => names.has(d.name.getText(file)))))
|
||||
.map(node => node.getText(file)).join('\n');
|
||||
return evaluate(code, {
|
||||
fixedChoiceSettings: new Map(),
|
||||
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
||||
setStringSetting: (key, value) => values.set(key, value),
|
||||
});
|
||||
}
|
||||
|
||||
test('head-up preferences preserve untouched device fields and persist each explicit choice separately', () => {
|
||||
const values = new Map();
|
||||
let prefs = settings(values);
|
||||
assert.equal(prefs.getHeadUpPreferences().enabled, -1);
|
||||
assert.equal(prefs.getHeadUpPreferences().angle, -1);
|
||||
prefs.headUpWakeSetting.set(false);
|
||||
prefs = settings(values);
|
||||
assert.equal(prefs.headUpWakeSetting.get(), false);
|
||||
assert.equal(prefs.getHeadUpPreferences().enabled, 0);
|
||||
assert.equal(prefs.getHeadUpPreferences().angle, -1);
|
||||
prefs.headUpAngleSetting.set('75');
|
||||
prefs = settings(values);
|
||||
assert.equal(prefs.headUpAngleSetting.get(), '60');
|
||||
assert.equal(prefs.getHeadUpPreferences().enabled, 0);
|
||||
assert.equal(prefs.getHeadUpPreferences().angle, 60);
|
||||
for (const [input, expected] of [['-5', '0'], ['20.8', '21'], ['NaN', '30'], ['', '30']]) {
|
||||
assert.equal(prefs.headUpAngleSetting.set(input), expected);
|
||||
}
|
||||
assert.equal(values.has('glanceboard.showOnHeadTilt'), false);
|
||||
});
|
||||
|
||||
test('controller skips unset preferences, deduplicates changes and forces reconnect synchronization', async () => {
|
||||
const file = ts.createSourceFile('controller.ts', source('app/g2/dashboard-controller.ts'), ts.ScriptTarget.Latest, true);
|
||||
const controller = file.statements.find(node => ts.isClassDeclaration(node) && node.name.text === 'DashboardController');
|
||||
const method = controller.members.find(node => node.name?.getText(file) === 'pushHeadUpPreferences').getText(file);
|
||||
let prefs = { enabled: -1, angle: -1 };
|
||||
const { Harness } = evaluate(`export class Harness { ${method} }`, {
|
||||
global: { isAndroid: true }, getHeadUpPreferences: () => prefs,
|
||||
});
|
||||
const h = new Harness(), calls = [];
|
||||
h.communicator = { configureHeadUpWake: async (...args) => calls.push(args) };
|
||||
h.pushHeadUpPreferences(); assert.equal(calls.length, 0);
|
||||
prefs = { enabled: 0, angle: 25 };
|
||||
h.pushHeadUpPreferences(); h.pushHeadUpPreferences();
|
||||
assert.deepEqual(calls, [[0, 25]]);
|
||||
h.pushHeadUpPreferences(true); assert.equal(calls.length, 2);
|
||||
prefs = { enabled: 1, angle: 40 }; h.pushHeadUpPreferences();
|
||||
assert.deepEqual(calls.at(-1), [1, 40]);
|
||||
let failures = 0;
|
||||
h.communicator.configureHeadUpWake = async () => { throw new Error('offline'); };
|
||||
h.appendLog = () => failures++;
|
||||
h.formatError = error => error.message;
|
||||
h.pushHeadUpPreferences(true);
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
assert.equal(h.lastPushedHeadUp, null); assert.equal(failures, 1);
|
||||
const connect = controller.members.find(node => node.name?.getText(file) === 'connect').getText(file);
|
||||
assert.match(connect, /pushHeadUpPreferences\(true\)/);
|
||||
});
|
||||
@@ -6,8 +6,7 @@ const ts = require('typescript');
|
||||
const path = require('node:path');
|
||||
// The provider settings ios-preview-controller hands the voice bridge.
|
||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||
getVoiceTranscriptionLanguage: () => '',
|
||||
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||
function load(file, context) {
|
||||
@@ -169,7 +168,6 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
|
||||
let cachedFont = 'Light';
|
||||
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
|
||||
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
||||
setTimeout: fn => tasks.push(fn),
|
||||
});
|
||||
@@ -386,7 +384,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
||||
recordTo: name => { bridge.recording = name; },
|
||||
setRawSession: session => { bridge.rawSession = session; } };
|
||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||
saveTranscribeAudioSetting: { get: () => true } };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||
|
||||
@@ -316,6 +316,33 @@ private class Session {
|
||||
}
|
||||
|
||||
class GlassesSessionCoreTest {
|
||||
@Test fun headUpWritesOnlyExplicitFieldsToBothArmsAndResendsAfterReconnect() {
|
||||
val s = Session()
|
||||
try {
|
||||
s.startAndAwaitLayout()
|
||||
fun writes() = s.link.writes.filter { it.kind == "head-up-control" }
|
||||
assertEquals(0, writes().size, "untouched firmware preferences must not be written")
|
||||
s.core.configureHeadUpWake(0, 30)
|
||||
assertTrue(waitUntil(5_000) { writes().size == 4 })
|
||||
for (arm in listOf(LEFT, RIGHT)) {
|
||||
val commands = writes().filter { it.address == arm }
|
||||
assertContentEquals(BleProtocol.buildSetHeadUpAngle(commands[0].magic, 30), commands[0].pb)
|
||||
assertContentEquals(BleProtocol.buildSetHeadUpWake(commands[1].magic, false), commands[1].pb)
|
||||
}
|
||||
s.core.handleTransportFailure("head-up reconnect test")
|
||||
assertTrue(waitUntil(5_000) { !s.layoutCreated() })
|
||||
s.platform.offsetMs += 10_000
|
||||
s.core.interruptibleSleep.interrupt()
|
||||
assertTrue(waitUntil(5_000) { s.layoutCreated() && writes().size == 8 }, s.host.logs().takeLast(10).toString())
|
||||
s.core.configureHeadUpWake(-1, 100)
|
||||
assertTrue(waitUntil(5_000) { writes().size == 10 })
|
||||
assertTrue(writes().takeLast(2).all { it.label == "head-up angle=60" })
|
||||
s.core.configureHeadUpWake(1, -1)
|
||||
assertTrue(waitUntil(5_000) { writes().size == 12 })
|
||||
assertTrue(writes().takeLast(2).all { it.label == "head-up wake=true" })
|
||||
} finally { s.core.close() }
|
||||
}
|
||||
|
||||
@Test fun brightnessConfigPrecedesFirstFrameAndSleepWakeFollowsComposites() {
|
||||
val s = Session()
|
||||
s.core.configureBrightness(false, 70, 2, 100, "0:0,1000:100", 280)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertContentEquals
|
||||
|
||||
class HeadUpProtocolTest {
|
||||
private fun bytes(hex: String) = hex.chunked(2).map { it.toInt(16).toByte() }.toByteArray()
|
||||
|
||||
@Test fun separateSwitchAndAngleMessagesMatchReferenceWireFormat() {
|
||||
// G2SettingPackage{1=1, 2=magic, 3={4=HeadUpSetting{1=switch | 2=angle}}}.
|
||||
assertContentEquals(bytes("080110071a0422020800"), BleProtocol.buildSetHeadUpWake(7, false))
|
||||
assertContentEquals(bytes("080110071a0422020801"), BleProtocol.buildSetHeadUpWake(7, true))
|
||||
assertContentEquals(bytes("080110071a042202101e"), BleProtocol.buildSetHeadUpAngle(7, 30))
|
||||
assertContentEquals(bytes("080110071a0422021000"), BleProtocol.buildSetHeadUpAngle(7, -10))
|
||||
assertContentEquals(bytes("080110071a042202103c"), BleProtocol.buildSetHeadUpAngle(7, 100))
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class WhisperModelFilesTest {
|
||||
@Test fun existingEnglishFilesRemainUsable() {
|
||||
for (language in listOf(null, "en", " EN ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertTrue(model.englishOnly)
|
||||
assertEquals("en", model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
|
||||
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
|
||||
for (language in listOf("fr", "de", "ja", " FR ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals(language.trim().lowercase(), model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
|
||||
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
|
||||
val model = WhisperModelFiles.forLanguage("")
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals("", model.language)
|
||||
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,6 @@
|
||||
"../app/ui/menu-core.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/native/java-direct-buffer.ts",
|
||||
"../app/native/transcription-language.ts",
|
||||
"../app/native/notification-types.ts",
|
||||
"../app/g2/firmware-compat.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
|
||||
@@ -163,8 +163,7 @@ function androidBridge() {
|
||||
starts: 0,
|
||||
setListener(listener) { this.listener = listener; },
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
|
||||
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||
stop() { this.active = false; },
|
||||
isCapturing() { return this.active; },
|
||||
@@ -172,8 +171,7 @@ function androidBridge() {
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null,
|
||||
transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: function () { return controller; },
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
@@ -181,26 +179,6 @@ function androidBridge() {
|
||||
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
||||
}
|
||||
|
||||
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.modelKind, 'whisper');
|
||||
assert.equal(controller.language, 'fr');
|
||||
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.starts, 1);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
|
||||
assert.equal(controller.language, '');
|
||||
assert.equal(controller.starts, 2);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
||||
// An omitted language preserves the native bridge's original English behavior.
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.stop();
|
||||
});
|
||||
|
||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
const options = { provider: 'onboard-whisper' };
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, requireModule = () => ({}), globals = {}) {
|
||||
const exports = {};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
|
||||
return exports;
|
||||
}
|
||||
const languages = load('app/native/transcription-language.ts');
|
||||
function settings(values = new Map()) {
|
||||
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
|
||||
: name.includes('settings-store') ? {
|
||||
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
||||
setStringSetting: (key, value) => values.set(key, value),
|
||||
getBooleanSetting: (_key, fallback) => fallback,
|
||||
onSettingsStoreChanged() {},
|
||||
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
|
||||
}
|
||||
|
||||
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
const s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
|
||||
});
|
||||
|
||||
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
let s = settings(values);
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
|
||||
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
|
||||
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
|
||||
}
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
|
||||
s.voiceLanguageSetting.set('auto');
|
||||
s = settings(values);
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
s.voiceLanguageSetting.set('default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
});
|
||||
|
||||
test('corrupt language preferences revert to the compatible provider defaults', () => {
|
||||
const s = settings(new Map([['voice.language', 'invalid']]));
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
});
|
||||
|
||||
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
|
||||
const s = settings();
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s.voiceProviderSetting.set('whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
for (const [file, name, method] of [
|
||||
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
||||
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
||||
]) {
|
||||
const host = { continuousClients: new Map() };
|
||||
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
|
||||
// Run each real capture-options method with only its native dependencies stubbed.
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
|
||||
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
||||
const capture = loadBody(member.body.getText(tree), s, host);
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
||||
assert.equal(capture().provider, 'soniox');
|
||||
assert.equal(capture().language, 'ja');
|
||||
host.continuousClients.clear();
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
}
|
||||
});
|
||||
|
||||
function loadBody(body, s, host) {
|
||||
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
|
||||
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
|
||||
}
|
||||
|
||||
test('multilingual downloads and deletion leave an existing English model intact', () => {
|
||||
const files = new Map(), downloads = [];
|
||||
const asr = load('app/native/asr-model.ts', () => ({
|
||||
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
||||
}), {
|
||||
java: { io: { File: class {
|
||||
constructor(name) { this.name = name; }
|
||||
exists() { return files.has(this.name); }
|
||||
length() { return files.get(this.name) ?? 0; }
|
||||
delete() { return files.delete(this.name); }
|
||||
} } },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
||||
FaceclawModelDownloader: class {
|
||||
constructor(url, destination, hash, size, listener) {
|
||||
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
||||
}
|
||||
start() {}
|
||||
cancel() {}
|
||||
},
|
||||
} } },
|
||||
});
|
||||
const english = asr.ASR_MODELS['whisper-base-en'];
|
||||
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
asr.startAsrModelDownload('whisper-base');
|
||||
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
|
||||
for (let index = 0; index < 3; index++) {
|
||||
const download = downloads[index];
|
||||
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
|
||||
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
||||
files.set(download.destination, download.size);
|
||||
download.listener.onDone();
|
||||
}
|
||||
assert.equal(downloads.length, 3);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), true);
|
||||
asr.deleteAsrModel('whisper-base');
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
});
|
||||
|
||||
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
||||
const values = new Map();
|
||||
let s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
||||
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
});
|
||||
Reference in new issue
Block a user