forked from Blendman974/faceclaw
Translate app: sound level meter; translate unfinished segments
This commit is contained in:
1 parent
bf2addd287
commit
0b7d93a3f1
25 files changed
+679
-175
No files matched your search
@@ -74,9 +74,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -2116,10 +2116,10 @@ class DashboardController {
|
||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||
}
|
||||
|
||||
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
|
||||
private multilingualProvider(): VoiceProvider | undefined {
|
||||
/** The provider and language a continuous-capture window needs (Translate's), if any. */
|
||||
private transcriptionChoice(): TranscriptionChoice | undefined {
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if (options.multilingualProvider) return options.multilingualProvider;
|
||||
if (options.transcription) return options.transcription;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -2144,20 +2144,21 @@ class DashboardController {
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs a multilingual provider sets it for every capture
|
||||
// while it is open, so push-to-talk still shares its stream.
|
||||
const multilingual = this.multilingualProvider();
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: multilingual ?? voiceProviderSetting.get(),
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
// Whisper detects the language itself when the speech may be in any.
|
||||
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
|
||||
@@ -149,11 +149,12 @@ export class IosPreviewController {
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Windows that want continuous capture, and what each needs of it (a
|
||||
// recording, a multilingual provider). They share one capture, which runs
|
||||
// recording, a provider and language). They share one capture, which runs
|
||||
// while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
||||
// transcriptionKey of the running continuous capture's provider; one that no
|
||||
// longer matches what its windows need is drained and restarted.
|
||||
// transcriptionKey of the running continuous capture (provider and
|
||||
// language); one that no longer matches what its windows need is drained
|
||||
// and restarted.
|
||||
private continuousTranscription = ''
|
||||
// Continuous capture should (re)start once audio is available: a window
|
||||
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||
@@ -1058,15 +1059,17 @@ export class IosPreviewController {
|
||||
}
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs a multilingual provider (Translate) sets it for every
|
||||
* capture while it is open, with Whisper left to detect the language.
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
|
||||
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
Reference in new issue
Block a user