diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index ec143d1a..eb6068a6 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -136,8 +136,6 @@ class FaceclawVoiceController(context: Context) { override fun release() {} } } - - override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) } } private val session = VoiceCaptureSession(host) @@ -155,11 +153,6 @@ class FaceclawVoiceController(context: Context) { session.setUsePhoneMic(usePhoneMic) } - /** When true, the decoded mic PCM for each session is saved as a WAV. */ - fun setSaveRecordings(saveRecordings: Boolean) { - session.setSaveRecordings(saveRecordings) - } - /** * Which on-device model {@link #start}("onboard") should load: "whisper" * selects the second on-device model (sherpa-onnx offline Whisper); any @@ -233,29 +226,6 @@ class FaceclawVoiceController(context: Context) { stop() } - /** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */ - private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) { - try { - val dir = File(appContext.getExternalFilesDir(null), "voice-recordings") - if (!dir.exists() && !dir.mkdirs()) { - Log.w(TAG, "could not create voice-recordings dir") - return - } - val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US) - .format(java.util.Date()) - val file = File(dir, "voice-$stamp.wav") - java.io.FileOutputStream(file).use { fos -> - fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1)) - fos.write(pcmBytes) - } - Log.i(TAG, "saved voice recording " + file.absolutePath - + " samples=" + (pcmBytes.size / 2) - + " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate)) - } catch (t: Throwable) { - Log.w(TAG, "failed to save voice recording", t) - } - } - /** * The on-device model directory for the given kind, populated by the * download flow in asr-model.ts (releases before 0.5.0 copied the diff --git a/app/apps/glanceboard/widgets/system-card.ts b/app/apps/glanceboard/widgets/system-card.ts index 90fb5056..95cb103a 100644 --- a/app/apps/glanceboard/widgets/system-card.ts +++ b/app/apps/glanceboard/widgets/system-card.ts @@ -11,6 +11,7 @@ import { ensureLocationPermission, hasLocationPermission } from "../../../native import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons"; import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types"; import { readPhoneBatteryState } from "../../../native/phone-battery"; +import { voiceControlBridge } from "../../../native/voice-control"; import { weatherBridge } from "../../../native/weather"; import { formatClockDate, formatClockTime } from "../../../ui/clock-format"; import { @@ -26,6 +27,7 @@ import { watchBatteryVisibilitySetting, type BatteryIndicatorVisibility, } from "../../../ui/dashboard-settings"; +import { RECORDING_ICON } from "../../../ui/listening-header"; import { type MenuItem } from "../../../ui/menu"; import { shell } from "../../../ui/shell/shell"; import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness"; @@ -149,13 +151,16 @@ function requestWeatherLocationAccess(): void { * off, and the layout mirrored either way, in the card's submenu of * Glanceboard > Settings. Battery style and per-device visibility follow * Settings > Customization > Battery indicators, so the card agrees with - * the bar. Countdowns tick once a second only while one is running. + * the bar. Countdowns tick once a second only while one is running. While + * Transcribe or Translate is saving the mic's audio (their Save Recording), + * the recording circle sits under the batteries and weather. */ export class SystemCardWidget implements GlanceWidget { private requestRender: (() => void) | null = null; private minuteTimer: ReturnType | null = null; private secondTimer: ReturnType | null = null; private unsubscribeNotifications: (() => void) | null = null; + private unsubscribeRecording: (() => void) | null = null; private unsubscribeSettings: (() => void) | null = null; private unsubscribeTimers: (() => void) | null = null; private unsubscribeWeather: (() => void) | null = null; @@ -164,6 +169,7 @@ export class SystemCardWidget implements GlanceWidget { this.requestRender = requestRender; this.scheduleMinuteTick(); this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.()); + this.unsubscribeRecording = voiceControlBridge.onRecordingChange(() => this.requestRender?.()); this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.()); this.unsubscribeTimers = timerEngine.onChange(() => { this.syncSecondTick(); @@ -182,6 +188,8 @@ export class SystemCardWidget implements GlanceWidget { this.secondTimer = null; this.unsubscribeNotifications?.(); this.unsubscribeNotifications = null; + this.unsubscribeRecording?.(); + this.unsubscribeRecording = null; this.unsubscribeSettings?.(); this.unsubscribeSettings = null; this.unsubscribeTimers?.(); @@ -237,6 +245,8 @@ export class SystemCardWidget implements GlanceWidget { if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems)); const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null; if (weather) sideBlocks.push(weather); + // Last, so it comes and goes without moving the others. + if (voiceControlBridge.recording) sideBlocks.push(RECORDING_BLOCK); const side: Rect[] = []; let sideOffset = PAD; for (const block of sideBlocks) { @@ -382,6 +392,13 @@ function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock { }; } +/** The recording circle: the mic's audio is being saved (see RECORDING_ICON). */ +const RECORDING_BLOCK: SideBlock = { + width: RECORDING_ICON.width, + height: RECORDING_ICON.height, + draw: (image, left, top) => image.drawImage(RECORDING_ICON, left, top), +}; + /** The current weather's condition icon and temperature, or null without a fresh reading. */ function weatherBlock(font: UiFont, nowMs: number): SideBlock | null { const reading = weatherReading(weatherBridge.snapshot(), nowMs); diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index 900af357..f175e818 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -1,4 +1,7 @@ -import { imageFromAsciiArt } from "../../graphics/image"; +import { imageFromAsciiArt, type GrayImage } from "../../graphics/image"; +import { voiceControlBridge } from "../../native/voice-control"; +import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { withRecordingIcon } from "../../ui/listening-header"; import { TranscribeLayer, transcriptTimestamp } from "./transcribe"; import { createInProcessWindow, @@ -13,8 +16,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe"; const TRAY_ICON_ID = "transcribe"; -// 16x18 microphone glyph for the top-bar tray (also Translate's). -export const MIC_ICON = imageFromAsciiArt( +// 14x15 microphone glyph for the top-bar tray (also Translate's). +const MIC_ICON = imageFromAsciiArt( [ " #### ", " ###### ", @@ -34,6 +37,12 @@ export const MIC_ICON = imageFromAsciiArt( ], { "#": 220 }, ); +const MIC_RECORDING_ICON = withRecordingIcon(MIC_ICON); + +/** The listening apps' tray icon: a microphone, with the recording circle while their audio is saved. */ +export function micTrayIcon(recording: boolean): GrayImage { + return recording ? MIC_RECORDING_ICON : MIC_ICON; +} export type TranscribeAppOptions = InProcessAppOptions & { /** Begin continuous mic capture (kept on while the window is open). */ @@ -59,18 +68,23 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc closeable: true, actions: options.actions, baseLayer: new YieldAtRootLayer(layer), - menuItems: () => [{ - label: "Save", - onSelect: (ctx) => { - ctx.stack.pop(); - layer.saveTranscript(); + menuItems: () => [ + { + label: "Save", + onSelect: (ctx) => { + ctx.stack.pop(); + layer.saveTranscript(); + }, }, - }], + // The controller starts and stops the recording as the setting changes. + toggleSettingMenuItem(saveRecordingSetting), + ], submitFrame: options.submitFrame, setSurfaceVisible: options.setSurfaceVisible, removeSurface: options.removeSurface, onClosed: () => { layer.onRemoved(); + unsubscribeRecording(); options.stopContinuousVoiceCapture(); shell.setTrayIcon(TRAY_ICON_ID, null); options.onClosed(); @@ -79,7 +93,11 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID)); + const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => { + shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording)); + app.requestRender(); + }); options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); - shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); + shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording)); return app; } diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 36df07a2..05e1fef7 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -1,7 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; -import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; +import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -50,8 +50,11 @@ export class TranscribeLayer implements Layer { const viewport = ctx.stack.getBaseSize(); const image = new GrayImage(viewport.width, viewport.height, 0); - const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null, - viewport.padding); + const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status; + const bodyTop = paintListeningHeader( + image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding, + voiceControlBridge.recording, + ); const step = lineStep(font) + 2; const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step); const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); diff --git a/app/apps/translate/translate-app.ts b/app/apps/translate/translate-app.ts index dd0800ab..455a0aaa 100644 --- a/app/apps/translate/translate-app.ts +++ b/app/apps/translate/translate-app.ts @@ -7,11 +7,12 @@ import { translateProviderSetting, translateShowOriginalSetting, } from "./translate-settings"; -import { MIC_ICON } from "../transcribe/transcribe-app"; +import { micTrayIcon } from "../transcribe/transcribe-app"; import { transcriptTimestamp } from "../transcribe/transcribe"; +import { voiceControlBridge } from "../../native/voice-control"; import { type ContinuousCaptureOptions } from "../../ui/layers"; import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel"; -import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings"; import { createInProcessWindow, YieldAtRootLayer, @@ -44,6 +45,11 @@ export type TranslateAppOptions = InProcessAppOptions & { * provider (Translate settings), which a shared capture switches to. */ export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow { + // One timestamp per session names both the audio recording and a saved transcript. + const stamp = transcriptTimestamp(); + // Whether the mic tray icon is up (capture was asked for). + let trayIconShown = false; + const syncTrayIcon = () => shell.setTrayIcon(TRAY_ICON_ID, trayIconShown ? micTrayIcon(voiceControlBridge.recording) : null); // Checks the setup and (re)starts capture with the chosen provider and // language: at launch, when either setting changes, and on a tap after a // problem. @@ -51,15 +57,18 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces const provider = resolveTranslateProvider(); const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null; layer.setProblem(problem); - if (!provider || problem) { + trayIconShown = Boolean(provider) && !problem; + if (provider && !problem) { + options.startContinuousVoiceCapture({ + recordAs: `translate-${stamp}`, + transcription: { provider, language: resolveTranslateLanguage() }, + }); + } else { options.stopContinuousVoiceCapture(); - shell.setTrayIcon(TRAY_ICON_ID, null); - return; } - options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } }); - shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); + syncTrayIcon(); }; - const layer = new TranslateLayer(transcriptTimestamp(), startCapture); + const layer = new TranslateLayer(stamp, startCapture); const app = createInProcessWindow({ appId: "translate", windowId: TRANSLATE_WINDOW_ID, @@ -77,6 +86,8 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces layer.saveTranscript(); }, }, + // The controller starts and stops the recording as the setting changes. + toggleSettingMenuItem(saveRecordingSetting), { label: "Translate settings", onSelect: (ctx) => { @@ -94,6 +105,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces removeSurface: options.removeSurface, onClosed: () => { layer.onRemoved(); + unsubscribeRecording(); options.stopContinuousVoiceCapture(); shell.setTrayIcon(TRAY_ICON_ID, null); options.onClosed(); @@ -102,6 +114,10 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID)); + const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => { + syncTrayIcon(); + app.requestRender(); + }); startCapture(); return app; } diff --git a/app/apps/translate/translate.ts b/app/apps/translate/translate.ts index cb826b31..18d52592 100644 --- a/app/apps/translate/translate.ts +++ b/app/apps/translate/translate.ts @@ -1,7 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; -import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; +import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; @@ -76,9 +76,13 @@ export class TranslateLayer implements Layer { const image = new GrayImage(viewport.width, viewport.height, 0); const measure = (line: string) => font.measureText(line); - const status = this.problem ? "Setup needed" : this.translationError || this.status; + const status = this.problem + ? "Setup needed" + : voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.translationError || this.status; const meter = this.listening && !this.problem ? this.meter : null; - const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, viewport.padding); + const bodyTop = paintListeningHeader( + image, font, "Translate", this.saveNotice || status, meter, viewport.padding, voiceControlBridge.recording, + ); const step = lineStep(font) + 2; const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step); const maxWidth = layout.textWidth; diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 78bca5c1..8732bc73 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -78,7 +78,7 @@ import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarSt import { sameEdges } from "../ui/viewport"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -488,6 +488,7 @@ class DashboardController { if (!resized && !appsResized) this.applyWindowEdgesIfChanged(); this.syncAssistantBridgeIfChanged(); this.syncLockScreenSettingIfChanged(); + this.syncContinuousRecording(); // A Preview color or scaling change should show on the mirror at once, // not at the next composited frame (cheap: floor-limited, dedup'd on // the Java side). @@ -2194,12 +2195,14 @@ class DashboardController { private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { this.continuousClients.set(client, options); + this.syncContinuousRecording(); this.retargetContinuousCapture(); this.ensureContinuousCapture(true); } private stopContinuousVoiceCapture(client: string): void { if (!this.continuousClients.delete(client)) return; + this.syncContinuousRecording(); if (this.continuousClients.size > 0) { // The window that needed its own provider may be the one that left. this.retargetContinuousCapture(); @@ -2225,6 +2228,18 @@ class DashboardController { void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim()); } + /** + * Record to the first client's requested name while Save Recording is on + * (it is toggled from those windows' menus, so this also runs when a + * setting changes). + */ + private syncContinuousRecording(): void { + const name = saveRecordingSetting.get() + ? Array.from(this.continuousClients.values(), (options) => options.recordAs).find(Boolean) ?? "" + : ""; + voiceControlBridge.recordTo(name); + } + /** Move a running continuous capture onto the provider its windows now call for. */ private retargetContinuousCapture(): void { voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator)); @@ -2274,7 +2289,6 @@ class DashboardController { whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get(), - saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances // against the enrolled wearer voice-print; non-matching speakers' // finals are suppressed by the bridge. diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 60b1c881..9842a207 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -53,7 +53,7 @@ import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, - whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings' + whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveRecordingSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' @@ -418,6 +418,7 @@ export class IosPreviewController { this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { this.syncLockSetting() + this.syncContinuousRecording() this.pushBrightness() this.relayout() shell.foregroundWindow()?.requestRender() @@ -1111,7 +1112,7 @@ export class IosPreviewController { * Continuous capture on behalf of `client` (one id per window: Transcribe, * Translate, Teleprompter, Calculator); it runs while any client wants it, as * on Android. `recordAs` names a WAV of the audio in Downloads when Save - * Transcribe audio is on. + * Recording is on. */ private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { this.continuousClients.set(client, options) @@ -1200,9 +1201,12 @@ export class IosPreviewController { private syncContinuousClaim(): void { voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) } - /** Record to the first client's requested name while Save Transcribe audio is on. */ + /** + * Record to the first client's requested name while Save Recording is on (it is + * toggled from those windows' menus, so this also runs when a setting changes). + */ private syncContinuousRecording(): void { - const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : '' + const name = saveRecordingSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : '' iosVoiceInput.recordTo(name) } /** diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 407ab3cd..9d8f3864 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -62,6 +62,9 @@ export class IosVoiceControlBridge { // and it is created by the first audio, so a failed start leaves no file. private recordingName = '' private recorder: WavRecorder | null = null + // The file could not be created: one attempt per recording, not one per packet. + private recordingFailure = false + private readonly recordingListeners = new Set<() => void>() // Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to // rawListeners while any owner wants it, from the session setRawSession // bound. It shares the one glasses mic with speech capture, which preempts @@ -212,16 +215,28 @@ export class IosVoiceControlBridge { recordTo(name: string): void { if (name === this.recordingName) return this.recorder?.close(); this.recorder = null - this.recordingName = name + this.setRecording(name, false) + } + /** Whether continuous captures' audio is being saved (see recordTo). */ + get recording(): boolean { return this.recordingName !== '' && !this.recordingFailure } + /** Whether the recording asked for could not be saved (its file could not be created). */ + get recordingFailed(): boolean { return this.recordingFailure } + /** `recording` or `recordingFailed` changed. */ + onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => { this.recordingListeners.delete(listener) } } + private setRecording(name: string, failed: boolean): void { + const wasRecording = this.recording, hadFailed = this.recordingFailure + this.recordingName = name; this.recordingFailure = failed + if (wasRecording === this.recording && hadFailed === failed) return + for (const fn of [...this.recordingListeners]) fn() } private record(pcm: Uint8Array): void { - if (!this.recorder && this.recordingName) { + if (!this.recording) return + if (!this.recorder) { this.recorder = WavRecorder.open(this.recordingName) this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') - // One attempt per recording, not one per packet. - if (!this.recorder) this.recordingName = '' + if (!this.recorder) { this.setRecording(this.recordingName, true); return } } - this.recorder?.append(pcm) + this.recorder.append(pcm) } /** End continuous capture, transcribing the audio since the last pause. */ stopContinuousCapture(): Promise { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index d6a7e37b..eaae562d 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core"; import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { SpeechPauseDetector } from "./speech-pause"; +import { WavRecorder } from "./wav-recorder"; import { toUint8Array } from "../util/array-util"; import { pcmLevel } from "../util/audio-level"; @@ -60,7 +61,6 @@ export type PushToTalkOptions = { whisperServerApiKey?: string; /** Spoken language for cloud providers (see VoiceProviderSelection.language). */ language?: string; - saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free * ("Hey Even") capture, which has no button release to end the utterance. @@ -150,6 +150,14 @@ export class FaceclawVoiceControlBridge { // the bridge swallow it. private verificationActive = false; private verificationRejected = false; + // Continuous captures' audio goes to Downloads/.wav while it is + // set. The file outlives a capture, so a restarted one keeps appending to it, + // and it is created by the first audio, so a failed start leaves no file. + private recordingName = ""; + private recorder: WavRecorder | null = null; + // The file could not be created: one attempt per recording, not one per packet. + private recordingFailure = false; + private readonly recordingListeners = new Set<() => void>(); onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); @@ -242,6 +250,56 @@ export class FaceclawVoiceControlBridge { this.acquireCapture("continuous", options); } + /** + * Record continuous captures' audio to Downloads/.wav, across restarts, + * until called with another name or "". + */ + recordTo(name: string): void { + if (name === this.recordingName) return; + this.recorder?.close(); + this.recorder = null; + this.setRecording(name, false); + } + + /** Whether continuous captures' audio is being saved (see recordTo). */ + get recording(): boolean { + return this.recordingName !== "" && !this.recordingFailure; + } + + /** Whether the recording asked for could not be saved (its file could not be created). */ + get recordingFailed(): boolean { + return this.recordingFailure; + } + + /** `recording` or `recordingFailed` changed. */ + onRecordingChange(listener: () => void): () => void { + this.recordingListeners.add(listener); + return () => this.recordingListeners.delete(listener); + } + + private setRecording(name: string, failed: boolean): void { + const wasRecording = this.recording; + const hadFailed = this.recordingFailure; + this.recordingName = name; + this.recordingFailure = failed; + if (wasRecording === this.recording && hadFailed === failed) return; + for (const listener of Array.from(this.recordingListeners)) listener(); + } + + /** `pcm` is the controller's Java byte[], handed to the recorder as it came. */ + private record(pcm: any): void { + if (!this.recording) return; + if (!this.recorder) { + this.recorder = WavRecorder.open(this.recordingName); + if (!this.recorder) { + this.setRecording(this.recordingName, true); + return; + } + console.log(`Voice: recording audio to ${this.recorder.path}`); + } + this.recorder.append(pcm); + } + /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ onRawPcm(listener: RawPcmListener): () => void { this.rawPcmListeners.add(listener); @@ -305,7 +363,6 @@ export class FaceclawVoiceControlBridge { // The raw tap is strictly the G2 stream; clear any phone-mic flag a // preview-mode capture may have left on the shared controller. this.controller?.setUsePhoneMic(false); - this.controller?.setSaveRecordings(false); this.controller?.setEndpointing(false); // Raw means raw: the Microphones session (and EvenHub apps) get the // unprocessed stream and run their own beam/suppression processing. @@ -354,7 +411,6 @@ export class FaceclawVoiceControlBridge { this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); - this.controller?.setSaveRecordings(options.saveRecording); this.controller?.setEndpointing(Boolean(options.endpointing)); this.captureEndpointing = Boolean(options.endpointing); this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression)); @@ -554,6 +610,8 @@ export class FaceclawVoiceControlBridge { onPcm: (pcm: any) => { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); + // Push-to-talk on a capture of its own is not recorded. + if (this.captureHolders.has("continuous")) this.record(pcm); if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) { for (const listener of this.speechPauseListeners) listener(); } diff --git a/app/native/wav-recorder.ts b/app/native/wav-recorder.ts new file mode 100644 index 00000000..cec96f3f --- /dev/null +++ b/app/native/wav-recorder.ts @@ -0,0 +1,42 @@ +declare const android: any; +declare const java: any; +declare const com: any; + +const SAMPLE_RATE = 16000; + +/** + * Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory + * while it is captured (Android; wav-recorder.ios.ts is the iOS one). The + * native recorder keeps the header's sizes current, so a crash leaves a + * playable file of everything up to the last second. + */ +export class WavRecorder { + private constructor(private recorder: any, readonly path: string) {} + + /** A recorder for Downloads/.wav (a -2, -3... suffix if taken), or null. */ + static open(name: string): WavRecorder | null { + try { + const downloads = android.os.Environment.getExternalStoragePublicDirectory( + android.os.Environment.DIRECTORY_DOWNLOADS, + ); + downloads.mkdirs(); + let file = new java.io.File(downloads, `${name}.wav`); + for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`); + const path = String(file.getAbsolutePath()); + return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path); + } catch (error) { + console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`); + return null; + } + } + + /** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */ + append(pcm: any): void { + this.recorder?.append(pcm); + } + + close(): void { + this.recorder?.finish(); + this.recorder = null; + } +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index c65ff5a9..d47a981b 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -1045,20 +1045,16 @@ export const wakeWordActionSetting = new ConfigSettingEnum({ description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.", }); -export const saveTranscribeAudioSetting = new ConfigSettingBoolean({ - id: "save-transcribe-audio", - label: "Save Transcribe audio", - storageKey: "voice.saveTranscribeAudio", - defaultValue: true, - description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.", -}); - -export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({ - id: "save-voice-recordings", - label: "Save voice recordings", - storageKey: "developer.saveVoiceRecordings", +/** + * Toggled from the Transcribe and Translate apps' menus. One setting for + * both: they share the mic stream, so its audio is either being saved or not. + */ +export const saveRecordingSetting = new ConfigSettingBoolean({ + id: "save-recording", + label: "Save Recording", + storageKey: "voice.saveRecording", defaultValue: false, - description: "Keep a copy of captured voice audio on the phone, for debugging transcription problems.", + description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.", }); export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 03b7984f..8785caad 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -70,7 +70,6 @@ import { ringConnectionModeSetting, glassesTypeSetting, sonioxApiKeySetting, - saveTranscribeAudioSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, @@ -81,7 +80,6 @@ import { lockScreenEnabledSetting, menuAnimationSetting, newNotificationDisplaySetting, - saveVoiceRecordingsSetting, showBleBandwidthSetting, suspendEvenHubWhenScreenOffSetting, temperatureUnitSetting, @@ -250,7 +248,6 @@ function settingsSections(): SettingsSection[] { // Whether and how the phone opens its own BLE link to the R1 ring. // Applied at connect time. ringConnectionMenuItem(), - toggleSettingMenuItem(saveVoiceRecordingsSetting), toggleSettingMenuItem(firmwareDebugFlagsSetting), toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting), toggleSettingMenuItem(useMicControlSetting), @@ -297,7 +294,7 @@ function settingsSections(): SettingsSection[] { toggleSettingMenuItem(showBleBandwidthSetting)] }; // No downloadable on-device models: iOS's on-device option is Apple's recognizer. if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), - enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; + enumSettingMenuItem(voiceProviderSetting), ...whisperServerMenuItems()] }; if (section.label === "Display") return { ...section, items: section.items.filter(item => item.label !== screenTimeoutSetting.label) }; return section; diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 1c010619..f41f3acf 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -13,7 +13,7 @@ export type TextSettingsEditToggle = { /** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */ export type ContinuousCaptureOptions = { - /** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */ + /** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */ recordAs?: string; /** * Transcribe with this provider, expecting this spoken language (ISO 639-1, diff --git a/app/ui/listening-header.ts b/app/ui/listening-header.ts index ffbb764a..4c334d19 100644 --- a/app/ui/listening-header.ts +++ b/app/ui/listening-header.ts @@ -1,4 +1,4 @@ -import { type GrayImage, type UiFont } from "../graphics/image"; +import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image"; import { truncateText } from "../graphics/textwrap"; import { voiceControlBridge } from "../native/voice-control"; import { lineStep } from "./metrics"; @@ -11,6 +11,10 @@ const SEGMENTS = 8; const SEGMENT_WIDTH = 4; const SEGMENT_GAP = 2; const METER_GAP = 10; +/** Between the recording circle and the meter. */ +const RECORDING_ICON_GAP = 4; +/** Between a tray icon and its recording circle: tight, so they read as one item in the bar. */ +const TRAY_RECORDING_ICON_GAP = 2; const LIT_GRAY = 200; const UNLIT_GRAY = 45; /** How often the meter may ask for a repaint (each one is a frame over BLE). */ @@ -18,6 +22,38 @@ const TICK_MS = 200; /** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */ const DECAY = 0.5; +/** The listening views' status while Save Recording is on but its file could not be created. */ +export const RECORDING_FAILED_NOTICE = "Recording failed (check file access)."; + +/** + * 9x9 filled circle: the mic's audio is being saved (Save Recording, in the + * Transcribe and Translate menus). Shown beside the mic level meter and the + * mic tray icon, and on Glanceboard's system card. + */ +export const RECORDING_ICON = imageFromAsciiArt( + [ + " ### ", + " ####### ", + " ####### ", + "#########", + "#########", + "#########", + " ####### ", + " ####### ", + " ### ", + ], + { "#": 220 }, +); + +/** `icon` with the recording circle to its right: a tray icon while its app's audio is saved. */ +export function withRecordingIcon(icon: GrayImage): GrayImage { + const circleX = icon.width + TRAY_RECORDING_ICON_GAP; + const image = new GrayImage(circleX + RECORDING_ICON.width, Math.max(icon.height, RECORDING_ICON.height), 0); + image.bitBlt(icon, 0, (image.height - icon.height) >> 1); + image.bitBlt(RECORDING_ICON, circleX, (image.height - RECORDING_ICON.height) >> 1); + return image; +} + /** * A compact segmented mic-level meter for listening views. Levels arrive per * 50 ms audio chunk; the meter holds their peak and, at most every 200 ms, @@ -65,8 +101,9 @@ export class MicLevelMeter { /** * The one-line header of the listening views (Transcribe, Translate): * "Title · status", the status truncated to fit, then the mic level meter at - * the line's end when one is given (while listening). It spans the image - * inside `padding`, the padding suggested for the window's edges + * the line's end when one is given (while listening), with the recording + * circle before it while the audio it shows is being saved. It spans the + * image inside `padding`, the padding suggested for the window's edges * (viewport.ts). Returns the y at which the view's body starts. */ export function paintListeningHeader( @@ -76,14 +113,23 @@ export function paintListeningHeader( status: string, meter: MicLevelMeter | null, padding: Edges, + recording = false, ): number { const x = padding.left, y = padding.top; let textRight = image.width - padding.right; if (meter) { const meterX = textRight - meter.width; // Roughly cap height, sitting on the text's baseline. - meter.paint(image, meterX, y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6))); + const baseline = y + font.ascent; + const meterHeight = Math.max(4, Math.round(font.ascent * 0.6)); + meter.paint(image, meterX, baseline, meterHeight); textRight = meterX - METER_GAP; + if (recording) { + const iconX = meterX - RECORDING_ICON_GAP - RECORDING_ICON.width; + // Centred on the meter's height. + image.drawImage(RECORDING_ICON, iconX, Math.round(baseline - (meterHeight + RECORDING_ICON.height) / 2)); + textRight = iconX - METER_GAP; + } } image.drawText(font, x, y, title, TITLE_GRAY); if (status) { diff --git a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt index ed002424..fabf7d49 100644 --- a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt +++ b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/FaceclawWavRecorder.kt @@ -40,6 +40,10 @@ constructor( private var file: RandomAccessFile? private var dataBytes = 0L + // The PCM bytes the header on disk accounts for. It is rewritten about once a second of + // audio, so a process death mid-recording still leaves a playable file. + private var headerDataBytes = 0L + init { val parent = File(path).parentFile if (parent != null && !parent.exists() && !parent.mkdirs()) @@ -58,11 +62,26 @@ constructor( try { current.write(pcm16le) dataBytes += pcm16le.size + if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current) } catch (e: IOException) { Log.w("FaceclawWavRecorder", "append failed", e) } } + /** Rewrite the header to cover the PCM written so far, leaving the position at the end. */ + private fun writeHeader(target: RandomAccessFile) { + val header = + BinaryEncoding.wavHeader( + minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(), + sampleRate, + channels, + ) + target.seek(0) + target.write(header) + target.seek(header.size + dataBytes) + headerDataBytes = dataBytes + } + @Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L) fun getPath(): String = path @@ -72,16 +91,7 @@ constructor( val current = file ?: return dataBytes file = null try { - current.use { - it.seek(0) - it.write( - BinaryEncoding.wavHeader( - minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(), - sampleRate, - channels, - ) - ) - } + current.use { writeHeader(it) } } catch (e: IOException) { Log.w("FaceclawWavRecorder", "finish failed", e) } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt index 3a68f866..8ad954f0 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt @@ -35,14 +35,11 @@ interface VoiceCaptureHost { /** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */ fun speakerEmbedder(modelPath: String): SpeakerEmbedder - - /** Where a saved recording goes, or null when recordings cannot be stored. */ - fun recordingSink(): RecordingSink? } /** * Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late - * accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker + * accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker * verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with * REPLACE-semantics partials, and speaker verification at session end. The session owns its * worker thread (started through the shared [startThread]); listener callbacks go through the @@ -158,11 +155,8 @@ class VoiceCaptureSession( private var lastTranscriptDecodeAtMs: Long = 0 private var lastTranscript = "" @Volatile - private var saveRecordings = false - @Volatile private var endpointing = false private val endpointDetector = VoiceEndpointDetector() - private var recordingPcm: ByteSink? = null @Volatile private var verifySpeakerModelPath: String? = null @Volatile @@ -199,10 +193,6 @@ class VoiceCaptureSession( this.usePhoneMic = usePhoneMic } - fun setSaveRecordings(saveRecordings: Boolean) { - this.saveRecordings = saveRecordings - } - fun setOnboardModelKind(kind: String?) { this.onboardModelKind = parseModelKind(kind) } @@ -363,7 +353,6 @@ class VoiceCaptureSession( } endpointDetector.reset() suppressor?.reset() - recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null verifyCount = 0 @@ -404,7 +393,6 @@ class VoiceCaptureSession( emitStatus("Voice control failed: " + error.message) } finally { stopG2Audio() - writeRecordingIfAny() releaseRecognizer() releaseDecoder() lock.withLock { @@ -418,25 +406,6 @@ class VoiceCaptureSession( } } - private fun appendRecording(pcm: ShortArray, count: Int) { - val out = recordingPcm ?: return - out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count)) - } - - private fun writeRecordingIfAny() { - val out = recordingPcm - recordingPcm = null - if (out == null || out.size() == 0) { - return - } - val sink = host.recordingSink() ?: return - try { - sink.save(out.toByteArray(), SAMPLE_RATE) - } catch (t: Throwable) { - logWarn("failed to save voice recording (${t.message})") - } - } - private fun startG2Audio(): Boolean { val source = host.g2AudioSource() ?: return false resetAudioStats() @@ -489,9 +458,6 @@ class VoiceCaptureSession( if (suppressionEnabled) { applySuppression(pcm, count) } - if (recordingPcm != null) { - appendRecording(pcm, count) - } val verifyBuffer = this.verifyBuffer if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt index 1844459a..2c0bdd9a 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt @@ -34,11 +34,6 @@ fun interface CallbackDispatcher { fun post(action: () -> Unit) } -/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */ -fun interface RecordingSink { - fun save(pcm16le: ByteArray, sampleRate: Int) -} - /** * A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared * [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus. diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index b6ce78e2..fa2004a4 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -365,6 +365,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => { const calls = [], claims = [], providers = []; + let saveRecording = false; // Models the bridge's continuous state: listening on a source, or draining after a stop. const bridge = { source: null, draining: false, recording: '', rawSession: undefined, supportsContinuous: voice => voice.provider !== 'onboard', @@ -382,7 +383,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec setRawSession: session => { bridge.rawSession = session; } }; const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' }, whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' }, - saveTranscribeAudioSetting: { get: () => true } }; + saveRecordingSetting: { get: () => saveRecording } }; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings : id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } @@ -401,7 +402,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle(); host.startContinuousVoiceCapture('calculator'); await settle(); assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture'); + // Save Recording (in the window's menu) starts and stops the recording while it is open. + assert.equal(bridge.recording, ''); + saveRecording = true; host.syncContinuousRecording(); assert.equal(bridge.recording, 'transcribe-1'); + saveRecording = false; host.syncContinuousRecording(); + assert.equal(bridge.recording, ''); + saveRecording = true; host.syncContinuousRecording(); + assert.equal(bridge.recording, 'transcribe-1'); + assert.deepEqual(calls, [['glasses', true]], 'without touching the capture'); assert.deepEqual(claims.at(-1), [true, 'continuous-capture']); host.stopContinuousVoiceCapture('calculator'); assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening'); diff --git a/tests/ios-voice.test.cjs b/tests/ios-voice.test.cjs index 66c92440..2561c249 100644 --- a/tests/ios-voice.test.cjs +++ b/tests/ios-voice.test.cjs @@ -31,8 +31,13 @@ function harness() { }, }; const recordings = []; + let failRecordings = false, recordingAttempts = 0; class FakeRecorder { - static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; } + static open(name) { + recordingAttempts++; + if (failRecordings) return null; + const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; + } append(bytes) { this.pcm.push(...bytes); } close() { this.closed = true; } } @@ -56,6 +61,7 @@ function harness() { } }; const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' }; return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(), + failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts, analyzerSupported: value => { analyzerSupported = value; }, onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' }, microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests, @@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => { const h = harness(); + let changes = 0; h.bridge.onRecordingChange(() => changes++); + assert.equal(h.bridge.recording, false); h.bridge.recordTo('transcribe-2026-10-17-120000'); + assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]); // A start that fails before any audio leaves no file behind. h.native.error = 'No phone microphone is available.'; await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true); @@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]); h.bridge.recordTo(''); assert.equal(h.recordings[0].closed, true); + assert.deepEqual([h.bridge.recording, changes], [false, 2]); +}); + +test('a recording whose file cannot be created is reported, and not retried per packet', async () => { + const h = harness(); + let changes = 0; h.bridge.onRecordingChange(() => changes++); + h.failRecordings(true); + h.bridge.recordTo('transcribe-1'); + const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start; + h.native.pcmHandler(new Uint8Array([1, 2])); + h.native.pcmHandler(new Uint8Array([3, 4])); + assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]); + // Asking for the same recording again changes nothing; another one (Save Recording + // switched off and on) is a fresh attempt. + h.bridge.recordTo('transcribe-1'); + h.native.pcmHandler(new Uint8Array([5])); + assert.equal(h.recordingAttempts(), 1); + h.failRecordings(false); + h.bridge.recordTo(''); + assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]); + h.bridge.recordTo('transcribe-1'); + h.native.pcmHandler(new Uint8Array([6])); + assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]); + assert.deepEqual(h.recordings[0].pcm, [6]); }); test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => { diff --git a/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/WavRecorderRepairTest.kt b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/WavRecorderRepairTest.kt index f35e7013..0034303b 100644 --- a/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/WavRecorderRepairTest.kt +++ b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/WavRecorderRepairTest.kt @@ -21,4 +21,31 @@ class WavRecorderRepairTest { file.delete() } } + + @Test + fun keepsTheHeaderCurrentWhileRecording() { + val file = File.createTempFile("recording", ".wav") + try { + val recorder = FaceclawWavRecorder(file.path, 16000, 1) + // Under a second of audio: still the empty header from the start. + recorder.append(ByteArray(16000)) + assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList()) + // Past a second the header covers what is on disk, without waiting for finish(), and + // later audio still lands at the end. + recorder.append(ByteArray(16000) { 1 }) + recorder.append(ByteArray(100) { 2 }) + var bytes = file.readBytes() + assertEquals(44 + 32100, bytes.size) + assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList()) + assertEquals(0.toByte(), bytes[44]) + assertEquals(1.toByte(), bytes[44 + 16000]) + assertEquals(2.toByte(), bytes.last()) + assertEquals(32100L, recorder.finish()) + bytes = file.readBytes() + assertEquals(44 + 32100, bytes.size) + assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList()) + } finally { + file.delete() + } + } } diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt index 17711f98..630d9bbc 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt @@ -63,7 +63,6 @@ class VoiceCaptureSessionTest { override val dispatcher = CallbackDispatcher { it() } var packetListener: FaceclawAudioPacketListener? = null var g2Stops = 0 - val recordings = mutableListOf() override fun onSessionStarting() {} override fun isG2SessionReady(): Boolean = g2Ready override fun isG2AudioCaptureActive(): Boolean = packetListener != null @@ -96,7 +95,6 @@ class VoiceCaptureSessionTest { override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding override fun release() {} } - override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) } } private val logs = mutableListOf() @@ -196,7 +194,6 @@ class VoiceCaptureSessionTest { val session = session(host, platform, "voice-test-onboard") session.setListener(events) session.setUsePhoneMic(true) - session.setSaveRecordings(true) session.start("onboard", 1) assertTrue(events.stoppedLatch.await(10000)) assertTrue(mic.closed) @@ -211,8 +208,6 @@ class VoiceCaptureSessionTest { // Every segment handed to the model is at most 8 s and was peak-normalized. assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES }) assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString()) - assertEquals(1, host.recordings.size) - assertEquals(170 * 800 * 2, host.recordings[0].size) assertEquals(170, events.pcmChunks.size) assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x")) assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y")) diff --git a/tests/listening-header.test.cjs b/tests/listening-header.test.cjs index 0c234b60..81247c6f 100644 --- a/tests/listening-header.test.cjs +++ b/tests/listening-header.test.cjs @@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts'); function harness() { const levelListeners = new Set(); let tick = null; - const { MicLevelMeter, paintListeningHeader } = loader({ + const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({ setInterval(fn) { tick = fn; return 1; }, clearInterval() { tick = null; }, }, { @@ -16,14 +16,15 @@ function harness() { } }, })('app/ui/listening-header.ts'); const level = value => { for (const listener of levelListeners) listener(value); }; - return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners }; + return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners }; } /** Records draws; text is 6px per character, like a fixed-width bitmap face. */ function fakeImage(width = 576) { - const texts = [], rects = []; - return { width, texts, rects, + const texts = [], rects = [], images = []; + return { width, texts, rects, images, drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); }, + drawImage(source, x, y) { images.push({ source, x, y }); }, fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } }; } const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 }; @@ -96,8 +97,48 @@ test('the header puts title and status on one line, truncated before the meter', // Segments sit on the text baseline. assert.ok(image.rects.every(rect => rect.y + rect.h === 8 + font.ascent)); + assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved'); + const idle = fakeImage(300); h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding); assert.equal(idle.rects.length, 0); assert.equal(idle.texts[1].text, ' · Starting microphone...'); }); + +test('while the audio is being saved, the recording circle sits beside the meter', () => { + const h = harness(); + const meter = new h.MicLevelMeter(); + const image = fakeImage(300); + const padding = { top: 8, right: 0, bottom: 8, left: 8 }; + h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, padding, true); + const meterX = Math.min(...image.rects.map(rect => rect.x)); + assert.equal(meterX, 300 - meter.width, 'the meter stays where it was'); + assert.equal(image.images.length, 1); + const [circle] = image.images; + assert.equal(circle.source, h.RECORDING_ICON); + assert.equal(circle.x + circle.source.width + 4, meterX); + // Centred on the meter, whose 7 rows sit on the text baseline. + const meterTop = Math.min(...image.rects.map(rect => rect.y)); + assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2); + const status = image.texts[1].text; + assert.ok(8 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle'); + + // The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark. + const idle = fakeImage(300); + h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding, true); + assert.equal(idle.images.length, 0); +}); + +test('a tray icon gains the recording circle to its right, vertically centred', () => { + const h = harness(); + const { GrayImage } = loader()('app/graphics/image.ts'); + const mic = new GrayImage(14, 15, 0); + mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200); + const icon = h.withRecordingIcon(mic); + assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]); + assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]); + assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them'); + // The circle's widest row is the icon's middle row; its corners are clear. + for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220); + assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]); +}); diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index c049d8b2..3050d7c8 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => { }); /** The Android voice bridge over a fake native controller (on-device recognition only). */ -function androidBridge() { +function androidBridge(modules = {}) { const controller = { starts: 0, setListener(listener) { this.listener = listener; }, - setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, + setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {}, setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {}, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, stop() { this.active = false; }, @@ -172,6 +172,7 @@ function androidBridge() { '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } }, './cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider }, + ...modules, }, { global: { isAndroid: true }, com: { faceclaw: { app: { FaceclawVoiceController: function () { return controller; }, FaceclawVoiceControllerListener: function (listener) { return listener; }, @@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is assert.equal(controller.active, true); assert.equal(bridge.holdsCapture('ptt'), true); }); + +test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => { + const recordings = []; + let fail = false, attempts = 0; + class FakeRecorder { + static open(name) { + attempts++; + if (fail) return null; + const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; + } + append(pcm) { this.pcm.push(...pcm); } + close() { this.closed = true; } + } + const { bridge, controller } = androidBridge({ + './wav-recorder': { WavRecorder: FakeRecorder }, + './speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } }, + '../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) }, + '../util/audio-level': { pcmLevel: () => 0 }, + }); + const options = { provider: 'onboard' }; + let changes = 0; bridge.onRecordingChange(() => changes++); + bridge.recordTo('transcribe-1'); + assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]); + // Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save. + bridge.startPushToTalk(options); + controller.listener.onPcm([9, 9]); + assert.equal(attempts, 0); + bridge.startContinuousCapture(options); + controller.listener.onPcm([1, 2]); + controller.listener.onPcm([3, 4]); + assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]); + // The file outlives a capture: one restarted after the glasses reconnect keeps appending to it. + bridge.handleSessionEnded(); + bridge.resumeCapture(options); + controller.listener.onPcm([5]); + assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]); + bridge.recordTo(''); + assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]); + controller.listener.onPcm([6]); + assert.equal(recordings[0].pcm.length, 5); + + // One attempt per recording, not one per packet; another recording is a fresh attempt. + fail = true; attempts = 0; + bridge.recordTo('transcribe-2'); + controller.listener.onPcm([7]); controller.listener.onPcm([8]); + assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]); + bridge.recordTo('transcribe-2'); + controller.listener.onPcm([9]); + assert.equal(attempts, 1); + fail = false; + bridge.recordTo(''); + bridge.recordTo('transcribe-2'); + controller.listener.onPcm([10]); + assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]); +});