forked from Blendman974/faceclaw
Add Translate app
This commit is contained in:
1 parent
07859c95ad
commit
c84d301c78
19 files changed
+926
-57
No files matched your search
@@ -73,9 +73,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -392,7 +392,8 @@ class DashboardController {
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||
startContinuousVoiceCapture: (client: string) => this.startContinuousVoiceCapture(client),
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
|
||||
this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
@@ -2066,21 +2067,28 @@ class DashboardController {
|
||||
return voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
// Windows that want continuous capture. They share one stream, which runs
|
||||
// while any of them wants it: one window letting go must not cut off another.
|
||||
private readonly continuousClients = new Set<string>();
|
||||
// Windows that want continuous capture, and what each needs of it. They
|
||||
// share one stream, which runs while any of them wants it: one window
|
||||
// letting go must not cut off another.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>();
|
||||
// The connection a continuous-capture start was last tried on (undefined:
|
||||
// none since the last teardown), so a window waiting on the glasses is
|
||||
// retried once per new session rather than on every frame.
|
||||
private continuousAttemptedOn: FaceclawCommunicatorBridge | null | undefined = undefined;
|
||||
|
||||
private startContinuousVoiceCapture(client: string): void {
|
||||
this.continuousClients.add(client);
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options);
|
||||
this.retargetContinuousCapture();
|
||||
this.ensureContinuousCapture(true);
|
||||
}
|
||||
|
||||
private stopContinuousVoiceCapture(client: string): void {
|
||||
if (!this.continuousClients.delete(client) || this.continuousClients.size > 0) return;
|
||||
if (!this.continuousClients.delete(client)) return;
|
||||
if (this.continuousClients.size > 0) {
|
||||
// The window that needed its own provider may be the one that left.
|
||||
this.retargetContinuousCapture();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
voiceControlBridge.stopContinuousCapture();
|
||||
} finally {
|
||||
@@ -2101,6 +2109,19 @@ class DashboardController {
|
||||
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
|
||||
}
|
||||
|
||||
/** Move a running continuous capture onto the provider its windows now call for. */
|
||||
private retargetContinuousCapture(): void {
|
||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||
}
|
||||
|
||||
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
|
||||
private multilingualProvider(): VoiceProvider | undefined {
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if (options.multilingualProvider) return options.multilingualProvider;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous capture's claim on the mic (the Microphones app and EvenHub
|
||||
* mic apps stand down while it is held) follows whether the capture really
|
||||
@@ -2121,16 +2142,20 @@ class DashboardController {
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs a multilingual provider sets it for every capture
|
||||
// while it is open, so push-to-talk still shares its stream.
|
||||
const multilingual = this.multilingualProvider();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: voiceProviderSetting.get(),
|
||||
provider: multilingual ?? voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(),
|
||||
// Whisper detects the language itself when the speech may be in any.
|
||||
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
|
||||
@@ -41,7 +41,7 @@ import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphi
|
||||
import { prepareFrameDraws } from '../graphics/glyph-wire'
|
||||
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
|
||||
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||
import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers'
|
||||
import { TextViewerLayer } from '../apps/files/text-viewer'
|
||||
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry'
|
||||
@@ -51,7 +51,7 @@ import { onAnySettingChanged,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
@@ -148,9 +148,13 @@ export class IosPreviewController {
|
||||
private acknowledgedFrames = 0
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Windows that want continuous capture, each with the recording it asked for
|
||||
// ('' for none). They share one capture, which runs while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, string>()
|
||||
// Windows that want continuous capture, and what each needs of it (a
|
||||
// recording, a multilingual provider). They share one capture, which runs
|
||||
// while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
||||
// transcriptionKey of the running continuous capture's provider; one that no
|
||||
// longer matches what its windows need is drained and restarted.
|
||||
private continuousTranscription = ''
|
||||
// Continuous capture should (re)start once audio is available: a window
|
||||
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||
// app lifecycle cut it short. A capture that fails doesn't set it, so a
|
||||
@@ -165,7 +169,7 @@ export class IosPreviewController {
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options?.recordAs),
|
||||
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
||||
@@ -965,15 +969,17 @@ export class IosPreviewController {
|
||||
}
|
||||
/**
|
||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||
* Teleprompter, Calculator); it runs while any client wants it, as on Android.
|
||||
* `recordAs` names a WAV of the audio in Downloads when Save Transcribe audio is on.
|
||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||
* Transcribe audio is on.
|
||||
*/
|
||||
private startContinuousVoiceCapture(client: string, recordAs = ''): void {
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.continuousClients.set(client, recordAs)
|
||||
this.syncContinuousRecording()
|
||||
// A listening capture is shared as it is; otherwise start one (after any drain).
|
||||
if (!iosVoiceInput.continuousSource) this.continuousPending = true
|
||||
@@ -982,7 +988,8 @@ export class IosPreviewController {
|
||||
private stopContinuousVoiceCapture(client: string): void {
|
||||
if (!this.continuousClients.delete(client)) return
|
||||
this.syncContinuousRecording()
|
||||
if (this.continuousClients.size > 0) return
|
||||
// The window that needed its own provider may be the one that left.
|
||||
if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return }
|
||||
// A start still awaiting permission or the mic backs off.
|
||||
++this.continuousCaptureToken
|
||||
this.continuousPending = false
|
||||
@@ -1009,8 +1016,10 @@ export class IosPreviewController {
|
||||
const glassesReady = !!this.communicator && this.state.phase === 'connected'
|
||||
if (iosVoiceInput.continuousActive) {
|
||||
// Draining or listening already. A phone capture from before the glasses
|
||||
// connected is drained here, and its end restarts it on the glasses.
|
||||
if (glassesReady && iosVoiceInput.continuousSource === 'phone') {
|
||||
// connected is drained here, and its end restarts it on the glasses; so
|
||||
// is one transcribing with another provider than its windows now need.
|
||||
const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider())
|
||||
if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) {
|
||||
this.continuousPending = true
|
||||
void iosVoiceInput.stopContinuousCapture()
|
||||
}
|
||||
@@ -1023,6 +1032,7 @@ export class IosPreviewController {
|
||||
private async beginContinuousCapture(): Promise<void> {
|
||||
const voice = this.voiceProvider()
|
||||
if (!iosVoiceInput.supportsContinuous(voice)) return
|
||||
this.continuousTranscription = transcriptionKey(voice)
|
||||
// A stop during an await below bumps the token; this start then backs off.
|
||||
const token = ++this.continuousCaptureToken
|
||||
const current = () => token === this.continuousCaptureToken
|
||||
@@ -1043,15 +1053,20 @@ export class IosPreviewController {
|
||||
}
|
||||
/** Record to the first client's requested name while Save Transcribe audio is on. */
|
||||
private syncContinuousRecording(): void {
|
||||
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].find(Boolean) ?? '' : ''
|
||||
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
|
||||
iosVoiceInput.recordTo(name)
|
||||
}
|
||||
/** The transcription provider setting and the keys the cloud ones need. */
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs a multilingual provider (Translate) sets it for every
|
||||
* capture while it is open, with Whisper left to detect the language.
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
|
||||
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
Reference in new issue
Block a user