forked from Blendman974/faceclaw
Voice: self-hosted Whisper server provider; Transcribe on iOS
- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests, with URL, model, language and key settings (Voice). Pause-ended spans are one request each, finalized in audio order, with provisional text while a span is open. See notes/whisper-server.md. - Transcribe (and other continuous capture) works on iOS with a cloud or Whisper-server provider; push-to-talk shares a running capture. - Transcribe saves each session's audio as transcribe-<time>.wav in Downloads (setting "Save Transcribe audio", on by default), written as it is captured so a crash leaves a playable file; a saved transcript uses the same stamp. - Calculator stops continuous listening when its window closes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ecb6c7f465
commit
e2caf28392
24 files changed
+1065
-43
No files matched your search
@@ -75,7 +75,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -2087,6 +2087,10 @@ class DashboardController {
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
// against the enrolled wearer voice-print; non-matching speakers'
|
||||
|
||||
@@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
||||
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
||||
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { voiceActivity } from '../ui/shell/voice-activity'
|
||||
import { nightscoutBridge } from '../native/nightscout-bridge'
|
||||
import { weatherBridge } from '../native/weather'
|
||||
import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios'
|
||||
@@ -48,12 +49,15 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
const SHELL_SURFACE_ID = 'shell'
|
||||
// voiceActivity holder for continuous capture (Transcribe), as on Android.
|
||||
const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture'
|
||||
const SHELL_SURFACE_Z_ORDER = 1
|
||||
const LOCK_SURFACE_Z_ORDER = 1000
|
||||
/** The shell render loop waits this long for the glasses to take a frame before moving on. */
|
||||
@@ -144,6 +148,7 @@ export class IosPreviewController {
|
||||
private acknowledgedFrames = 0
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private continuousCaptureToken = 0
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
||||
@@ -151,7 +156,8 @@ export class IosPreviewController {
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
||||
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
||||
@@ -941,10 +947,40 @@ export class IosPreviewController {
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||
}
|
||||
}
|
||||
/** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */
|
||||
private async startContinuousVoiceCapture(recordAs?: string): Promise<void> {
|
||||
if (this.glassesLocked) return
|
||||
const voice = this.voiceProvider()
|
||||
if (!iosVoiceInput.supportsContinuous(voice)) {
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
// A stop during an await below bumps the token; this start then backs off.
|
||||
const token = ++this.continuousCaptureToken
|
||||
const current = () => token === this.continuousCaptureToken
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : ''
|
||||
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
||||
const communicator = this.communicator
|
||||
if (communicator && this.state.phase === 'connected') {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record)
|
||||
} else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) {
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, false, true, record)
|
||||
}
|
||||
if (!current()) return
|
||||
if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
||||
}
|
||||
private stopContinuousVoiceCapture(): void {
|
||||
++this.continuousCaptureToken
|
||||
voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
||||
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
||||
}
|
||||
/** The transcription provider setting and the keys the cloud ones need. */
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() }
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
Reference in new issue
Block a user