From 9372b2742cd32dbd882cbda739ce3a0fda5a43fd Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 12 Nov 2026 21:54:57 -0800 Subject: [PATCH] Multi-mic support and replay mode for Transcribe app --- .../faceclaw/app/FaceclawVoiceController.kt | 102 +++ app/apps/files/files-app.ts | 16 +- app/apps/files/index.ts | 4 + app/apps/microphones/mic-control.ts | 21 + app/apps/transcribe/index.ts | 31 +- app/apps/transcribe/replay-time.ts | 23 + app/apps/transcribe/time-picker.ts | 147 ++++ app/apps/transcribe/transcribe-app.ts | 105 ++- app/apps/transcribe/transcribe-settings.ts | 40 + app/apps/transcribe/transcribe.ts | 85 +- app/apps/transcribe/transcript-model.ts | 12 + app/g2/dashboard-controller.ts | 53 +- app/g2/ios-preview-controller.ts | 4 +- app/native/mic-recorder.ios.ts | 50 ++ app/native/mic-recording.ts | 29 + app/native/voice-control.ios.ts | 13 +- app/native/voice-control.ts | 346 +++++++- app/native/wav-header.ts | 22 - app/native/wav-recorder.ios.ts | 75 -- app/native/wav-recorder.ts | 42 - app/ui/dashboard-settings.ts | 6 +- app/ui/layers.ts | 20 +- app/ui/listening-header.ts | 7 +- app/ui/shell/in-process-window.ts | 3 + app/ui/shell/shell.ts | 7 + native/kotlin/README.md | 3 +- .../com/faceclaw/app/PlatformUtil.android.kt | 38 + .../com/faceclaw/app/PlatformServices.kt | 30 + .../kotlin/com/faceclaw/app/audio/Matroska.kt | 764 ++++++++++++++++++ .../faceclaw/app/audio/MicArrayProtocol.kt | 93 +++ .../com/faceclaw/app/audio/MicRecording.kt | 484 +++++++++++ .../faceclaw/app/audio/VoiceCaptureSession.kt | 439 ++++++++-- .../com/faceclaw/app/audio/VoicePorts.kt | 18 + .../FaceclawVoiceControllerListener.kt | 9 + .../kotlin/com/faceclaw/app/IosMicRecorder.kt | 37 + .../com/faceclaw/app/PlatformUtil.ios.kt | 51 ++ tests/ios-voice.test.cjs | 4 +- .../com/faceclaw/app/MicRecordingFileTest.kt | 63 ++ .../com/faceclaw/app/MicRecordingTest.kt | 572 +++++++++++++ .../faceclaw/app/VoiceCaptureSessionTest.kt | 2 + .../com/faceclaw/app/IosMicRecorderTest.kt | 46 ++ tests/transcribe.test.cjs | 31 + tests/voice-finalization.test.cjs | 39 +- tests/voice-recording.test.cjs | 209 +++++ tests/wav-header.test.cjs | 19 - 45 files changed, 3926 insertions(+), 288 deletions(-) create mode 100644 app/apps/transcribe/replay-time.ts create mode 100644 app/apps/transcribe/time-picker.ts create mode 100644 app/apps/transcribe/transcribe-settings.ts create mode 100644 app/native/mic-recorder.ios.ts create mode 100644 app/native/mic-recording.ts delete mode 100644 app/native/wav-header.ts delete mode 100644 app/native/wav-recorder.ios.ts delete mode 100644 app/native/wav-recorder.ts create mode 100644 native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Matroska.kt create mode 100644 native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicArrayProtocol.kt create mode 100644 native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicRecording.kt create mode 100644 native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosMicRecorder.kt create mode 100644 tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/MicRecordingFileTest.kt create mode 100644 tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/MicRecordingTest.kt create mode 100644 tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosMicRecorderTest.kt create mode 100644 tests/voice-recording.test.cjs delete mode 100644 tests/wav-header.test.cjs diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index eb6068a6..239f08cc 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -114,6 +114,21 @@ class FaceclawVoiceController(context: Context) { } } + override fun micArrayLink(): MicArrayLink? { + val currentCommunicator = communicator ?: return null + return object : MicArrayLink { + override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean = + currentCommunicator.startG2AudioForwarding(listener) + + override fun stopForwarding() = currentCommunicator.stopG2AudioForwarding() + + override fun isForwarding(): Boolean = currentCommunicator.isAudioCaptureActive() + + override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean = + currentCommunicator.sendFaceclawMicControl(record, label, rightTemple, leftTemple) + } + } + override fun createPacketDecoder(): AudioPacketDecoder = if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter() else AndroidSpeechEngines.Lc3DecoderAdapter() @@ -140,6 +155,11 @@ class FaceclawVoiceController(context: Context) { private val session = VoiceCaptureSession(host) + // The recording of this controller's live captures (see startRecording). It outlives any one + // capture, and is only handed to the session while recording is enabled. + private var recording: MicRecordingWriter? = null + private var recordingEnabled = true + fun setListener(listener: FaceclawVoiceControllerListener?) { session.setListener(listener) } @@ -201,6 +221,88 @@ class FaceclawVoiceController(context: Context) { session.setBeamFilter(enabled, centerDeg, halfWidthDeg) } + /** + * Capture glasses sessions from the temples' own microphones (the custom firmware's + * mic_control stream) instead of the stock mixed stream. [configureRecord] is the CONFIGURE + * record to arm both temples with; null goes back to the mixed stream. Must be set before + * {@link #start}. + */ + fun setMicArray(configureRecord: ByteArray?) { + session.setMicArray(configureRecord) + } + + /** + * Have the next {@link #start} play the recording at [path] (a .mka file) from [startMs] in + * place of any microphone; null goes back to the microphones. Progress arrives through the + * listener's onReplay. + */ + fun setReplay(path: String?, startMs: Long) { + session.setReplay(path, startMs) + } + + /** Continue the running replay from [positionMs] into its recording. */ + fun seekReplay(positionMs: Long) { + session.seekReplay(positionMs) + } + + fun setReplayPaused(paused: Boolean) { + session.setReplayPaused(paused) + } + + /** + * Record live captures to a new Matroska audio file at [path], from now until + * {@link #stopRecording}, across however many captures run in between: every source's audio + * as it arrives, before any processing. [configJson] describes the capture and is stored in + * the file. Returns false when the file cannot be created. + */ + @Synchronized + fun startRecording(path: String, configJson: String?): Boolean { + stopRecording() + return try { + val parent = File(path).parentFile + if (parent != null && !parent.exists()) parent.mkdirs() + recording = MicRecordingWriter(openFileOutput(path), configJson, System.currentTimeMillis()) + session.setRecorder(if (recordingEnabled) recording else null) + true + } catch (t: Throwable) { + Log.w(TAG, "could not create recording $path", t) + recording = null + false + } + } + + /** + * Whether captures are written to the recording right now: off while the capture running + * isn't one that is to be recorded. + */ + @Synchronized + fun setRecordingEnabled(enabled: Boolean) { + recordingEnabled = enabled + session.setRecorder(if (enabled) recording else null) + } + + /** Whether a recording is open and still being written (false once writing to it failed). */ + @Synchronized + fun isRecording(): Boolean = recording?.failed == false + + /** + * Note a change in what is being captured (a JSON object) at the recording's current + * position; the audio after it is placed afresh on the recording's timeline. + */ + @Synchronized + fun noteRecordingEvent(json: String) { + recording?.event(json) + } + + /** Complete the recording's file. Returns its length in ms, or -1 when none was open. */ + @Synchronized + fun stopRecording(): Long { + val current = recording ?: return -1 + recording = null + session.setRecorder(null) + return current.finish() + } + fun start(requestedMode: String?) { session.start(requestedMode, 0) } diff --git a/app/apps/files/files-app.ts b/app/apps/files/files-app.ts index d8872e58..83abb779 100644 --- a/app/apps/files/files-app.ts +++ b/app/apps/files/files-app.ts @@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access"; import { isFontFile } from "../../native/font-files"; import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts"; import { isDecodableImageFile } from "../../native/image-files"; +import { isMicRecordingFile } from "../../native/mic-recording"; import { isPlayableVideoFile } from "../../native/video-player"; import { readEvenHubPackageManifest } from "../evenhub/installed-apps"; import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog"; @@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & { openFontWindow: (title: string, path: string) => void; /** Open a video file's player (settings panel first) as its own shell window. */ openVideoWindow: (title: string, path: string) => void; + /** Replay a recording (.mka) through Transcribe, in a window of its own. */ + openRecordingInTranscribe: (title: string, path: string) => void; }; /** @@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow const browser = new FileBrowserLayer({ isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || - (!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))), + (!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))), // The browser handles double-click itself (up a level), so it is not // wrapped in YieldAtRootLayer; it yields explicitly from the top level. onLeave: () => shell.yieldFocusToSidebar(), @@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI }, ]; } + if (isMicRecordingFile(entry.name)) { + return [ + { + label: "Open in Transcribe", + onSelect: (ctx) => { + ctx.stack.pop(); + options.openRecordingInTranscribe(entry.name, entry.path); + }, + }, + ]; + } if (isPlayableVideoFile(entry.name)) { return [ { diff --git a/app/apps/files/index.ts b/app/apps/files/index.ts index 65f9430f..984b51cc 100644 --- a/app/apps/files/index.ts +++ b/app/apps/files/index.ts @@ -14,6 +14,7 @@ import { FILES_WINDOW_ID, } from "./files-app"; import { createVideoPlayerWindow } from "./video-window"; +import { openRecordingInTranscribe } from "../transcribe"; // Document windows are closeable and non-singleton; the serial keeps their // windowIds unique. @@ -91,6 +92,9 @@ const filesApp: AppDefinition = { }, openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path), openVideoWindow: (title, path) => openVideoWindow(ctx, title, path), + openRecordingInTranscribe: (title, path) => { + openRecordingInTranscribe(ctx, path, title); + }, }), ), // A document arriving via Android's Share intent opens as its own window. diff --git a/app/apps/microphones/mic-control.ts b/app/apps/microphones/mic-control.ts index 6f62b3ad..ddc4a75d 100644 --- a/app/apps/microphones/mic-control.ts +++ b/app/apps/microphones/mic-control.ts @@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void { setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config)); } +/** + * The configuration a capture of the temples' own microphones arms them + * with: `mics` of 2 is one microphone per temple (the front one), 4 is both + * of each. The front end and direction detection follow Microphone setup + * (the Microphones app), except that both of a temple's microphones only + * come through the codec front end. Always raw PCM16 at 16 kHz, which is + * what the firmware streams and transcription takes. + */ +export function micArrayCaptureConfig(mics: 2 | 4): MicConfig { + const base = loadMicConfig(); + return { + ...base, + source: mics === 4 ? "codec" : base.source, + channelMask: mics === 4 ? 0b11 : 0b01, + codec: "raw", + format: "pcm16", + sampleRateHz: 16_000, + armHardware: true, + }; +} + class MicArrayController { private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>(); private statuses: MicArrayStatuses = { left: null, right: null }; diff --git a/app/apps/transcribe/index.ts b/app/apps/transcribe/index.ts index d5e504c6..ccdcddf9 100644 --- a/app/apps/transcribe/index.ts +++ b/app/apps/transcribe/index.ts @@ -1,5 +1,32 @@ -import { type AppDefinition } from "../app-definition"; -import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app"; +import { type AppContext, type AppDefinition } from "../app-definition"; +import { + createTranscribeAppWindow, + createTranscribeReplayWindow, + TRANSCRIBE_SURFACE_ID, + TRANSCRIBE_WINDOW_ID, +} from "./transcribe-app"; + +// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique. +let nextReplaySerial = 1; + +/** + * Open a recording (.mka) in Transcribe's replay mode, as a window of its + * own. Returns false where recordings can't be replayed. + */ +export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean { + if (!ctx.actions.openVoiceReplay) return false; + const windowId = `transcribe:replay:${nextReplaySerial++}`; + void ctx + .launchInProcessApp(windowId, `window:${windowId}`, (options) => { + const window = createTranscribeReplayWindow(windowId, path, name, options); + if (!window) throw new Error("recordings can't be replayed here"); + return window; + }) + .catch((error) => { + ctx.appendLog(`transcribe replay window failed: ${error}`); + }); + return true; +} const transcribeApp: AppDefinition = { appId: "transcribe", diff --git a/app/apps/transcribe/replay-time.ts b/app/apps/transcribe/replay-time.ts new file mode 100644 index 00000000..29c6268c --- /dev/null +++ b/app/apps/transcribe/replay-time.ts @@ -0,0 +1,23 @@ +/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */ +export function formatReplayTime(ms: number, withHours = false): string { + const total = Math.max(0, Math.floor(ms / 1000)); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const seconds = String(total % 60).padStart(2, "0"); + if (hours === 0 && !withHours) return `${minutes}:${seconds}`; + return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`; +} + +/** The time picker's fields, in display and click order. */ +export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const; +export type TimeField = (typeof TIME_FIELDS)[number]; + +const FIELD_SECONDS: Record = { hours: 3600, minutes: 60, seconds: 1 }; + +/** + * Step a position (whole seconds) by `delta` of a field's unit, carrying + * into the neighboring fields, within [0, maxSeconds]. + */ +export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number { + return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field])); +} diff --git a/app/apps/transcribe/time-picker.ts b/app/apps/transcribe/time-picker.ts new file mode 100644 index 00000000..3bb281e6 --- /dev/null +++ b/app/apps/transcribe/time-picker.ts @@ -0,0 +1,147 @@ +import { GrayImage } from "../../graphics/image"; +import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures"; +import type { Layer, LayerContext, PaintBelow } from "../../ui/layers"; +import { CONTEXT_MENU_DIM } from "../../ui/menu"; +import { centeredTextY } from "../../ui/metrics"; +import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time"; + +const PADDING = 12; +const FIELD_GAP = 6; +const FIELD_PAD_X = 6; +/** Fill and outline of the active field (the Timers and date pickers' field style). */ +const ACTIVE_FILL = 35; +const ACTIVE_STROKE = 130; + +export type TimePickerOptions = { + title: string; + /** Where the picker starts, and the furthest it goes, in ms. */ + initialMs: number; + maxMs: number; + onPick: (positionMs: number) => void; +}; + +/** + * A modal position picker for a recording: hours (when the recording has + * any), minutes and seconds, over the dimmed content beneath. Input follows + * the date picker's: scroll adjusts the active field (up is later, carrying + * into the fields beside it), click moves to the next field and on the last + * one picks, double-click steps back a field and on the first one cancels. + * A watch's swipes move between fields and adjust. + */ +export class TimePickerLayer implements Layer { + readonly acceptsDirectional = true; + readonly dimUnderneath = CONTEXT_MENU_DIM; + private readonly fields: readonly TimeField[]; + private readonly maxSeconds: number; + private seconds: number; + private field = 0; + + constructor(private readonly options: TimePickerOptions) { + this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000)); + this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000))); + this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1); + } + + handleInput(event: InputEvent, ctx: LayerContext): void { + switch (event.type) { + case "scroll-up": + case "swipe-up": + this.adjust(1); + return; + case "scroll-down": + case "swipe-down": + this.adjust(-1); + return; + case "swipe-right": + this.field = Math.min(this.fields.length - 1, this.field + 1); + return; + case "swipe-left": + this.field = Math.max(0, this.field - 1); + return; + case "click": + if (this.field < this.fields.length - 1) { + this.field++; + } else { + ctx.stack.pop(); + this.options.onPick(this.seconds * 1000); + } + return; + case "double-click": + if (this.field > 0) this.field--; + else ctx.stack.pop(); + return; + default: + return; + } + } + + private adjust(delta: number): void { + this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds); + } + + private hints(field: number): string { + return gestureHints([ + [GESTURE_SCROLL, "adjust"], + [GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"], + [GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"], + ]); + } + + paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { + const image = paintBelow(); + const small = getDefaultSmallFont(); + const medium = getDefaultMediumFont(); + const focused = ctx.stack.isFocused(); + const base = ctx.stack.getBaseSize(); + + const values = [ + String(Math.floor(this.seconds / 3600)), + String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"), + String(this.seconds % 60).padStart(2, "0"), + ].slice(TIME_FIELDS.length - this.fields.length); + // Sized for the widest value a field can show, so the dialog holds still while the time changes. + const fieldWidths = this.fields.map( + (field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X, + ); + const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1); + const boxHeight = medium.lineHeight + 6; + const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`; + const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index)))); + const width = Math.min( + base.width, + Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING, + ); + const titleHeight = small.lineHeight + 12; + const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight; + const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10); + const x = Math.max(0, Math.floor((base.width - width) / 2)); + const y = Math.max(0, Math.floor((base.height - height) / 2)); + + // Fill 1, not 0: 0 is the transparent color key on the shell surface. + image.fillRoundedRect(x, y, width, height, 1); + image.drawRoundedRect(x, y, width, height, 72); + image.drawText(small, x + PADDING, y + 8, this.options.title, 220); + + const top = y + 8 + titleHeight; + let fieldX = x + PADDING; + for (let index = 0; index < this.fields.length; index++) { + const fieldWidth = fieldWidths[index]!; + const active = index === this.field; + if (active) { + if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6); + image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6); + } + const value = values[index]!; + image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2), + centeredTextY(medium, top, boxHeight), value, active ? 250 : 170); + const caption = this.fields[index]!; + image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2), + top + boxHeight + 2, caption, active ? 190 : 110); + fieldX += fieldWidth + FIELD_GAP; + } + image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190); + image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110); + return image; + } +} diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index f175e818..1bd34659 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -1,8 +1,11 @@ import { imageFromAsciiArt, type GrayImage } from "../../graphics/image"; import { voiceControlBridge } from "../../native/voice-control"; -import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { type ContinuousCaptureOptions } from "../../ui/layers"; import { withRecordingIcon } from "../../ui/listening-header"; +import { TimePickerLayer } from "./time-picker"; import { TranscribeLayer, transcriptTimestamp } from "./transcribe"; +import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings"; import { createInProcessWindow, YieldAtRootLayer, @@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage { } export type TranscribeAppOptions = InProcessAppOptions & { - /** Begin continuous mic capture (kept on while the window is open). */ - startContinuousVoiceCapture: (options?: { recordAs?: string }) => void; + /** Begin (or retarget) continuous mic capture, kept on while the window is open. */ + startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void; stopContinuousVoiceCapture: () => void; }; @@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc // One timestamp per session names both the audio recording and a saved transcript. const stamp = transcriptTimestamp(); const layer = new TranscribeLayer(stamp); + // At launch, and again when the Microphones setting changes: a running + // capture moves over to the microphones now asked for. + const startCapture = () => + options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() }); const app = createInProcessWindow({ appId: "transcribe", windowId: TRANSCRIBE_WINDOW_ID, @@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc }, // The controller starts and stops the recording as the setting changes. toggleSettingMenuItem(saveRecordingSetting), + enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }), ], submitFrame: options.submitFrame, setSurfaceVisible: options.setSurfaceVisible, @@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording)); app.requestRender(); }); - options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); + startCapture(); shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording)); return app; } + +/** + * Transcribe in replay mode: a window of its own that transcribes a + * recording (a .mka file from Save Recording) in place of the microphones, + * in real time and through the same pipeline. It neither uses the mic nor + * shows the mic tray icon, so it runs beside a live Transcribe window. The + * header shows the position in the recording; a tap pauses and resumes; the + * menu jumps to a time. + */ +export function createTranscribeReplayWindow( + windowId: string, + path: string, + name: string, + options: InProcessAppOptions, +): InProcessWindow | null { + const replay = options.actions.openVoiceReplay?.(path) ?? null; + if (!replay) return null; + // Names a saved transcript after the recording it came from. + const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name); + const app = createInProcessWindow({ + appId: "transcribe", + windowId, + title: name, + iconLetter: "Tr", + icon: "mic", + closeable: true, + // Relaunching Transcribe after a restart would open the microphone, not this recording. + restoresApp: false, + // Like a playing video: something to watch, not to touch. + keepsScreenOn: () => replay.state === "playing", + actions: options.actions, + baseLayer: new YieldAtRootLayer(layer), + menuItems: () => [ + { + label: "Jump to time", + // The length isn't known until the recording has been opened. + disabled: () => replay.durationMs <= 0, + onSelect: (ctx) => { + ctx.stack.pop(); + ctx.stack.push( + new TimePickerLayer({ + title: "Jump to time", + initialMs: replay.positionMs, + maxMs: replay.durationMs, + onPick: (positionMs) => { + replay.jumpTo(positionMs); + app.requestRender(); + }, + }), + ); + }, + }, + { + label: replay.state === "paused" ? "Resume" : "Pause", + disabled: () => replay.state !== "playing" && replay.state !== "paused", + onSelect: (ctx) => { + ctx.stack.pop(); + replay.setPaused(replay.state === "playing"); + }, + }, + { + label: "Restart from 0:00", + onSelect: (ctx) => { + ctx.stack.pop(); + replay.jumpTo(0); + }, + }, + { + label: "Save", + onSelect: (ctx) => { + ctx.stack.pop(); + layer.saveTranscript(); + }, + }, + ], + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + onClosed: () => { + layer.onRemoved(); + replay.close(); + options.onClosed(); + }, + }); + // Started first, so the layer picks its opening status up as it subscribes. + replay.start(); + layer.start(app.requestRender, () => shell.isWindowVisible(windowId)); + return app; +} diff --git a/app/apps/transcribe/transcribe-settings.ts b/app/apps/transcribe/transcribe-settings.ts new file mode 100644 index 00000000..f9cc68f6 --- /dev/null +++ b/app/apps/transcribe/transcribe-settings.ts @@ -0,0 +1,40 @@ +import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings"; +import { type MicCount } from "../../ui/layers"; +import { micControlSupported } from "../microphones/mic-control"; + +export type TranscribeMics = "1" | "2" | "4"; + +const MICS_LABELS: Record = { + "1": "1 (mixed)", + "2": "2 (one per temple)", + "4": "4 (two per temple)", +}; + +/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */ +export function separateMicsSupported(): boolean { + return glassesTypeSetting.get() === "g2" && micControlSupported(); +} + +/** + * How many of the glasses' microphones Transcribe captures. With more than + * one, the temples stream their own microphones (each temple on its own + * clock) instead of the glasses' mixed stream; a recording keeps every one, + * and what is transcribed is a mix made on the phone. + */ +export const transcribeMicsSetting = new ConfigSettingEnum({ + id: "transcribe-mics", + label: "Microphones", + storageKey: "transcribe.mics", + defaultValue: "1", + values: ["1", "2", "4"], + formatValue: (value) => MICS_LABELS[value], + isDisabled: (value) => value !== "1" && !separateMicsSupported(), + description: + "Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.", +}); + +/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */ +export function transcribeMicCount(): MicCount { + if (!separateMicsSupported()) return 1; + return Number(transcribeMicsSetting.get()) as MicCount; +} diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 05e1fef7..78797709 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { lineStep } from "../../ui/metrics"; import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header"; import { writeTextToDownloads } from "../../native/file-access"; -import { voiceControlBridge } from "../../native/voice-control"; +import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control"; import { Layer, type LayerContext } from "../../ui/layers"; import { type InputEvent } from "../../ui/gestures"; +import { formatReplayTime } from "./replay-time"; import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model"; /** * Live transcription view. Continuous mic capture is owned by the app wrapper * (createTranscribeAppWindow); this layer accumulates the transcript, shows * it, and saves it to Downloads from the app menu. + * + * Given a replay, it transcribes that recording instead of the microphone: + * the header shows the position in the recording, a tap pauses and resumes, + * and each jump leaves its time as a line in the transcript. */ export class TranscribeLayer implements Layer { private status = "Starting microphone..."; @@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer { private unsubscribeTranscript: (() => void) | null = null; private unsubscribePause: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; + private unsubscribeReplay: (() => void) | null = null; + private readonly source: VoiceEventSource; - /** `stamp` names the saved transcript, matching the session's audio recording. */ - constructor(private readonly stamp: string = transcriptTimestamp()) {} + /** + * `stamp` names the saved transcript, matching the session's audio + * recording. `replay` is the recording to transcribe in place of the + * microphone; `replayName` is what to call it. + */ + constructor( + private readonly stamp: string = transcriptTimestamp(), + private readonly replay: VoiceReplay | null = null, + private readonly replayName = "", + ) { + this.source = replay ? replay.events : voiceControlBridge; + if (replay) this.status = "Opening the recording..."; + } - /** `isVisible` says whether the window is on screen (the level meter repaints only then). */ + /** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */ start(requestRender: () => void, isVisible: () => boolean): void { - this.meter.start(requestRender, isVisible); - this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { + this.meter.start(requestRender, isVisible, this.source); + this.unsubscribeTranscript = this.source.onTranscript((event) => { this.transcript.accept(event); requestRender(); }); - this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause()); - this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause()); + // The current status arrives while subscribing, before the window has a + // surface to render to; the window paints it when it is first shown. + let subscribing = true; + this.unsubscribeStatus = this.source.onStatus((state) => { this.status = state.status; this.listening = state.listening; this.detail = state.detail; this.saveNotice = ""; - requestRender(); + if (!subscribing) requestRender(); }); + subscribing = false; + const replay = this.replay; + if (replay) { + let shownSecond = -1; + let shownState = replay.state; + this.unsubscribeReplay = replay.onChange((jumpedToMs) => { + if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`); + // The clock ticks in whole seconds; repaint when what it shows changed. + const second = Math.floor(replay.positionMs / 1000); + if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return; + shownSecond = second; + shownState = replay.state; + if (isVisible()) requestRender(); + }); + } + } + + /** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */ + private replayStatus(replay: VoiceReplay): string { + if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status; + const withHours = replay.durationMs >= 3_600_000; + const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`; + if (replay.state === "paused") return `${clock} · Paused`; + if (replay.state === "ended") return `${clock} · Finished`; + // Anything but plain listening is worth showing: the model loading, a provider reconnecting. + return this.listening ? clock : `${clock} · ${this.status}`; } paint(ctx: LayerContext): GrayImage { @@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer { const viewport = ctx.stack.getBaseSize(); const image = new GrayImage(viewport.width, viewport.height, 0); - const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status; + const replay = this.replay; + const status = replay + ? this.replayStatus(replay) + : this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status; + const playing = !replay || replay.state === "playing"; const bodyTop = paintListeningHeader( - image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding, - voiceControlBridge.recording, + image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding, + this.source.recording, ); const step = lineStep(font) + 2; const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step); - const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); + const placeholder = replay ? `Replaying ${this.replayName}` : "Listening..."; + const text = this.transcript.text || (this.listening ? placeholder : this.detail); const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth); const firstLine = this.scroll.layout(wrapped.length, layout.pageLines); @@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer { ctx.actions.requestRender(); return; } + if (event.type === "click" && this.replay) { + this.replay.setPaused(this.replay.state === "playing"); + ctx.actions.requestRender(); + return; + } if (event.type === "double-click") { ctx.stack.pop(); } @@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer { onRemoved(): void { this.meter.stop(); + this.unsubscribeReplay?.(); + this.unsubscribeReplay = null; this.unsubscribePause?.(); this.unsubscribePause = null; this.unsubscribeTranscript?.(); diff --git a/app/apps/transcribe/transcript-model.ts b/app/apps/transcribe/transcript-model.ts index af950a70..05befc46 100644 --- a/app/apps/transcribe/transcript-model.ts +++ b/app/apps/transcribe/transcript-model.ts @@ -36,6 +36,18 @@ export class TranscriptModel { if (this.live) this.breakAfterLive = true; else if (this.finalized) this.separator = "\n"; } + + /** + * A line of its own (where a replay jumped to), ending whatever was in + * progress: what follows it starts a new paragraph. + */ + mark(label: string): void { + const before = this.text; + this.finalized = before ? `${before}\n${label}` : label; + this.live = ""; + this.separator = "\n"; + this.breakAfterLive = false; + } } /** An absolute history position, with an explicit follow-tail mode. */ diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 8732bc73..f743f003 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -53,7 +53,7 @@ import { type GlanceEvent } from "./glance-state"; import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; -import { voiceControlBridge } from "../native/voice-control"; +import { VoiceReplay, voiceControlBridge } from "../native/voice-control"; import { voiceActivity } from "../ui/shell/voice-activity"; import { GrayImage } from "../graphics/image"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; @@ -76,7 +76,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { sameEdges } from "../ui/viewport"; -import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; +import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; @@ -98,6 +98,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W import { isInstalledPackagePresent } from "../apps/evenhub/updates"; import { wearerVerificationOptions } from "../apps/microphones/speakers"; import { micSession } from "../apps/microphones/mic-session"; +import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control"; +import { encodeMicControl } from "../apps/microphones/mic-protocol"; import { chargingPreviewMessage, glassesDisplayAsleep, @@ -410,6 +412,7 @@ class DashboardController { startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => this.startContinuousVoiceCapture(client, options), stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client), + openVoiceReplay: (path: string) => this.openVoiceReplay(path), playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; @@ -2253,6 +2256,30 @@ class DashboardController { return undefined; } + /** + * How many of the glasses' microphones the continuous-capture windows call + * for (the most any asks for), where the glasses can stream them + * separately: G2 with the microphone channel in use. Otherwise 1, the + * mixed stream. + */ + private continuousMics(): MicCount { + if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1; + let mics: MicCount = 1; + for (const options of this.continuousClients.values()) { + if ((options.mics ?? 1) > mics) mics = options.mics!; + } + return mics; + } + + /** + * A recording as a capture of its own (see VoiceReplay), transcribed with + * the provider and processing settings a microphone capture would get. + * "My voice only" is left out: a recording is of whoever spoke. + */ + private openVoiceReplay(path: string): VoiceReplay { + return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined })); + } + /** * Continuous capture's claim on the mic (the Microphones app and EvenHub * mic apps stand down while it is held) follows whether the capture really @@ -2278,9 +2305,26 @@ class DashboardController { // Otherwise only Whisper server is told a language (its own setting). const choice = this.transcriptionChoice(); const provider = choice?.provider ?? voiceProviderSetting.get(); + // Separate microphones only come from connected glasses. + const mics = communicator ? this.continuousMics() : 1; + const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, + micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null, + // Stored in a recording of the capture (see Save Recording). + captureInfo: { + glasses: communicator ? glassesModelName() : null, + mics, + micConfig: micConfig && { + source: micConfig.source, + channelMask: micConfig.channelMask, + codec: micConfig.codec, + format: micConfig.format, + sampleRateHz: micConfig.sampleRateHz, + directionDetection: micConfig.beamform, + }, + }, provider, language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "", elevenLabsApiKey: elevenLabsApiKeySetting.get(), @@ -2721,10 +2765,11 @@ class DashboardController { if (!this.openAppsRestored || this.suppressOpenAppsPersist) return; const open: string[] = []; for (const window of shell.getWindows()) { - if (window.appId === "launcher") continue; + if (window.appId === "launcher" || window.restoresApp === false) continue; if (!open.includes(window.appId)) open.push(window.appId); } - savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null }); + const foreground = shell.foregroundWindow(); + savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null }); } /** diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 9842a207..2e605374 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -1111,8 +1111,8 @@ export class IosPreviewController { /** * Continuous capture on behalf of `client` (one id per window: Transcribe, * Translate, Teleprompter, Calculator); it runs while any client wants it, as - * on Android. `recordAs` names a WAV of the audio in Downloads when Save - * Recording is on. + * on Android. `recordAs` names a recording (.mka) of the audio in Downloads + * when Save Recording is on. */ private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { this.continuousClients.set(client, options) diff --git a/app/native/mic-recorder.ios.ts b/app/native/mic-recorder.ios.ts new file mode 100644 index 00000000..e5f76a3b --- /dev/null +++ b/app/native/mic-recorder.ios.ts @@ -0,0 +1,50 @@ +import { knownFolders, path as paths } from '@nativescript/core' + +// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt). +declare const FaceclawKitIosMicRecordings: any + +/** + * Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in + * Documents/Downloads while it is captured, through the shared Kotlin writer + * that Android's captures are recorded with. The file is written about a + * second at a time, so a crash leaves everything up to the last second. + */ +export class MicRecorder { + private constructor(private recorder: any, readonly path: string) {} + + /** + * A recorder for Downloads/.mka (a -2, -3... suffix if taken), or null. + * `info` describes the capture and is stored in the file. + */ + static open(name: string, info: Record = {}): MicRecorder | null { + const fm = NSFileManager.defaultManager + const dir = paths.join(knownFolders.documents().path, 'Downloads') + try { + fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null) + let path = paths.join(dir, `${name}.mka`) + for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`) + const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info)) + return recorder ? new MicRecorder(recorder, path) : null + } catch { + return null + } + } + + append(pcm: Uint8Array): void { + if (!this.recorder || !pcm.length) return + // The handle is the buffer's start: a view into a larger buffer is copied out first. + const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice() + try { + // Copied by the recorder before this returns. + this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)) + } catch { + // Stop recording rather than retry. + this.close() + } + } + + close(): void { + try { this.recorder?.finish() } catch { /* already closed */ } + this.recorder = null + } +} diff --git a/app/native/mic-recording.ts b/app/native/mic-recording.ts new file mode 100644 index 00000000..0c8663f1 --- /dev/null +++ b/app/native/mic-recording.ts @@ -0,0 +1,29 @@ +declare const android: any; +declare const java: any; + +/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */ +export const RECORDING_EXTENSION = ".mka"; + +export function isMicRecordingFile(name: string): boolean { + return name.toLowerCase().endsWith(RECORDING_EXTENSION); +} + +/** + * The path for a new recording in the public Downloads directory: + * Downloads/.mka, with a -2, -3... suffix if that is taken. Null when + * Downloads isn't available. The file itself is created by whoever records. + */ +export function newRecordingPath(name: string): string | null { + try { + const downloads = android.os.Environment.getExternalStoragePublicDirectory( + android.os.Environment.DIRECTORY_DOWNLOADS, + ); + downloads.mkdirs(); + let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`); + for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`); + return String(file.getAbsolutePath()); + } catch (error) { + console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`); + return null; + } +} diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 9d8f3864..aa0740a2 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt' import type { ReconnectingSttClient } from './reconnecting-stt' import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios' -import { WavRecorder } from './wav-recorder.ios' +import { MicRecorder } from './mic-recorder.ios' import { pcmLevel } from '../util/audio-level' declare const FaceclawSpeech: any @@ -57,11 +57,11 @@ export class IosVoiceControlBridge { private continuous = false // Shown instead of the native "ended" status once a drain the bridge started completes. private drainStatus: string | null = null - // Continuous captures' audio goes to Downloads/.wav while it is + // Continuous captures' audio goes to Downloads/.mka while it is // set. The file outlives a capture, so a restarted one keeps appending to it, // and it is created by the first audio, so a failed start leaves no file. private recordingName = '' - private recorder: WavRecorder | null = null + private recorder: MicRecorder | null = null // The file could not be created: one attempt per recording, not one per packet. private recordingFailure = false private readonly recordingListeners = new Set<() => void>() @@ -209,7 +209,7 @@ export class IosVoiceControlBridge { /** Where a listening continuous capture gets its audio; null when none is listening. */ get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null } /** - * Record continuous captures' audio to Downloads/.wav, across restarts, + * Record continuous captures' audio to Downloads/.mka, across restarts, * until called with another name or ''. Only cloud captures supply PCM. */ recordTo(name: string): void { @@ -232,7 +232,10 @@ export class IosVoiceControlBridge { private record(pcm: Uint8Array): void { if (!this.recording) return if (!this.recorder) { - this.recorder = WavRecorder.open(this.recordingName) + this.recorder = MicRecorder.open(this.recordingName, { + recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios', + source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1, + }) this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') if (!this.recorder) { this.setRecording(this.recordingName, true); return } } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index eaae562d..96243f7d 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,9 +1,9 @@ import { Utils } from "@nativescript/core"; -import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; +import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; +import { newRecordingPath } from "./mic-recording"; import { SpeechPauseDetector } from "./speech-pause"; -import { WavRecorder } from "./wav-recorder"; import { toUint8Array } from "../util/array-util"; import { pcmLevel } from "../util/audio-level"; @@ -84,6 +84,49 @@ export type PushToTalkOptions = { * frame). Null/absent leaves the mic omnidirectional. */ beamFilter?: { centerDeg: number; halfWidthDeg: number } | null; + /** + * Capture from the temples' own microphones (the custom firmware's + * mic_control stream) instead of the glasses' mixed stream. Each temple + * streams its microphones on its own link and clock; the phone mixes them + * into the mono feed that is transcribed. Null/absent keeps the mixed + * stream; ignored with usePhoneMic. + */ + micArray?: MicArrayCapture | null; + /** + * Play this recording (a .mka file from Save Recording) in real time in + * place of any microphone. The capture needs no glasses, and ends by + * itself when the recording does. + */ + replay?: { path: string; startMs: number } | null; + /** What the caller knows of the capture, for the description stored in a recording of it. */ + captureInfo?: Record; +}; + +export type MicArrayCapture = { + /** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */ + configure: Uint8Array; +}; + +/** Progress of a capture that replays a recording; positions are ms into it. */ +export type VoiceReplayEvent = { + /** + * `start` once the file is open, `position` about once per second played, + * `seek` when a jump took effect, `end` when the recording ran out, and + * `error` when it could not be opened (the status says why). + */ + kind: "start" | "position" | "seek" | "end" | "error"; + positionMs: number; + durationMs: number; +}; + +/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */ +export type VoiceEventSource = { + onStatus(listener: (state: VoiceControlState) => void): () => void; + onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void; + onSpeechPause(listener: () => void): () => void; + onMicLevel(listener: (level: number) => void): () => void; + readonly recording: boolean; + readonly recordingFailed: boolean; }; /** Who currently wants the mic running. */ @@ -130,9 +173,13 @@ export class FaceclawVoiceControlBridge { private captureEndpointing = false; // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: CloudSttClient | null = null; - // What the running STT stream transcribes with (transcriptionKey); a holder - // asking for something else restarts the stream rather than sharing it. + // What the running STT stream captures from and transcribes with + // (streamKey); a holder asking for something else restarts the stream + // rather than sharing it. private streamProvider: string | null = null; + // The options of the running stream, for the description of a recording of it. + private streamOptions: PushToTalkOptions | null = null; + private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>(); // Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the // controller runs in the decode-only "cloud" mode and every decoded PCM // frame is broadcast to these listeners. STT capture preempts it — a live @@ -150,11 +197,14 @@ export class FaceclawVoiceControlBridge { // the bridge swallow it. private verificationActive = false; private verificationRejected = false; - // Continuous captures' audio goes to Downloads/.wav while it is - // set. The file outlives a capture, so a restarted one keeps appending to it, - // and it is created by the first audio, so a failed start leaves no file. + // Continuous captures' audio goes to Downloads/.mka while it is + // set: every source's chunks as they arrive, before any processing, written + // by the controller. The file outlives a capture, so a restarted one keeps + // adding to it, and it is created by the first audio, so a failed start + // leaves no file. private recordingName = ""; - private recorder: WavRecorder | null = null; + // The file is open in the controller. + private recordingOpen = false; // The file could not be created: one attempt per recording, not one per packet. private recordingFailure = false; private readonly recordingListeners = new Set<() => void>(); @@ -251,16 +301,55 @@ export class FaceclawVoiceControlBridge { } /** - * Record continuous captures' audio to Downloads/.wav, across restarts, + * End the continuous capture as an utterance: stop it and have the + * transcript of its last audio come as a final. For a capture whose audio + * ran out by itself (a replay). + */ + finishContinuousCapture(): void { + this.releaseCapture("continuous", true); + } + + /** Progress of a capture that replays a recording (PushToTalkOptions.replay). */ + onReplay(listener: (event: VoiceReplayEvent) => void): () => void { + this.replayListeners.add(listener); + return () => this.replayListeners.delete(listener); + } + + /** Continue the running replay from `positionMs` into its recording. */ + seekReplay(positionMs: number): void { + this.controller?.seekReplay(Math.max(0, Math.round(positionMs))); + } + + /** Hold the running replay where it is, or let it go on. */ + setReplayPaused(paused: boolean): void { + this.controller?.setReplayPaused(paused); + } + + /** Whether a capture is running and still has audio coming (a replay stops being so at its end). */ + isCaptureLive(): boolean { + return this.micIsLive(); + } + + /** + * Record continuous captures' audio to Downloads/.mka, across restarts, * until called with another name or "". */ recordTo(name: string): void { if (name === this.recordingName) return; - this.recorder?.close(); - this.recorder = null; + this.closeRecording(); this.setRecording(name, false); } + private closeRecording(): void { + if (!this.recordingOpen) return; + this.recordingOpen = false; + try { + this.controller?.stopRecording(); + } catch (error) { + console.warn(`Voice: closing the recording failed: ${error}`); + } + } + /** Whether continuous captures' audio is being saved (see recordTo). */ get recording(): boolean { return this.recordingName !== "" && !this.recordingFailure; @@ -286,18 +375,48 @@ export class FaceclawVoiceControlBridge { for (const listener of Array.from(this.recordingListeners)) listener(); } - /** `pcm` is the controller's Java byte[], handed to the recorder as it came. */ - private record(pcm: any): void { - if (!this.recording) return; - if (!this.recorder) { - this.recorder = WavRecorder.open(this.recordingName); - if (!this.recorder) { - this.setRecording(this.recordingName, true); - return; - } - console.log(`Voice: recording audio to ${this.recorder.path}`); + /** + * Audio arrived for a continuous capture: open the recording asked for, if + * it isn't open yet. The controller writes it from then on, as the audio + * arrives on its own thread. + */ + private ensureRecording(): void { + if (!this.recording || this.recordingOpen) return; + const path = newRecordingPath(this.recordingName); + let opened = false; + try { + opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture()))); + } catch (error) { + console.warn(`Voice: could not create the recording: ${error}`); } - this.recorder.append(pcm); + if (!opened) { + this.setRecording(this.recordingName, true); + return; + } + this.recordingOpen = true; + console.log(`Voice: recording audio to ${path}`); + } + + /** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */ + private syncRecordingEnabled(): void { + this.controller?.setRecordingEnabled(this.captureHolders.has("continuous")); + } + + /** The running capture, as the JSON stored in a recording of it. */ + private describeCapture(): Record { + const options = this.streamOptions; + return { + recording: this.recordingName, + startedAt: new Date().toISOString(), + source: options?.usePhoneMic ? "phone" : "glasses", + micArray: Boolean(options?.micArray) && !options?.usePhoneMic, + transcription: { provider: options?.provider ?? "", language: options?.language ?? "" }, + processing: { + noiseSuppression: Boolean(options?.noiseSuppression), + beamFilter: options?.beamFilter ?? null, + }, + ...options?.captureInfo, + }; } /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ @@ -364,6 +483,8 @@ export class FaceclawVoiceControlBridge { // preview-mode capture may have left on the shared controller. this.controller?.setUsePhoneMic(false); this.controller?.setEndpointing(false); + this.controller?.setMicArray(null); + this.controller?.setReplay(null, 0); // Raw means raw: the Microphones session (and EvenHub apps) get the // unprocessed stream and run their own beam/suppression processing. this.controller?.setNoiseSuppression(false); @@ -399,18 +520,33 @@ export class FaceclawVoiceControlBridge { const micLive = this.micIsLive(); this.suspendedHolders.delete(holder); this.captureHolders.add(holder); - if (micLive && this.streamProvider === transcriptionKey(options)) { + if (micLive && this.streamProvider === streamKey(options)) { // Mic already running; the new holder just shares the existing stream // (transcripts are already broadcast to its listeners). + this.syncRecordingEnabled(); return; } - // Any leftover capture is dead, or transcribes with another provider than + // Any leftover capture is dead, or captures or transcribes otherwise than // this holder needs; drop it before starting the new one. if (this.started) this.teardownCapture(); - this.streamProvider = transcriptionKey(options); + this.streamProvider = streamKey(options); + this.streamOptions = options; this.ensureController(); + this.syncRecordingEnabled(); + // A recording under way carries on into this capture: note what changed. + if (this.recordingOpen) { + try { + this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() })); + } catch (error) { + console.warn(`Voice: recording event failed: ${error}`); + } + } this.controller?.setCommunicator(options.communicator); this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); + this.controller?.setMicArray( + options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null, + ); + this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0))); this.controller?.setEndpointing(Boolean(options.endpointing)); this.captureEndpointing = Boolean(options.endpointing); this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression)); @@ -435,7 +571,7 @@ export class FaceclawVoiceControlBridge { this.speechPause.reset(); // Replaces whatever the last capture left behind ("Listening...", an // error, the permission prompt) until the controller reports progress. - this.setStatus("Starting microphone..."); + this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone..."); // A previous push-to-talk commit may still be awaiting its final result. this.cloudClient?.stop(); const cloudClient = this.createCloudClient(options); @@ -480,6 +616,7 @@ export class FaceclawVoiceControlBridge { if (commit) this.cloudClient?.finish(); return; } + this.syncRecordingEnabled(); if (this.captureHolders.size > 0) { // Another holder still wants the mic; keep it running. return; @@ -504,6 +641,7 @@ export class FaceclawVoiceControlBridge { this.captureHolders.clear(); this.suspendedHolders.clear(); this.suspendedRaw = false; + this.syncRecordingEnabled(); this.teardownCapture(); this.setStatus("Voice control stopped."); } @@ -521,6 +659,7 @@ export class FaceclawVoiceControlBridge { this.suspendedHolders.add(holder); } this.captureHolders.clear(); + this.syncRecordingEnabled(); this.suspendedRaw = this.rawActive; this.teardownCapture(); if (this.suspendedHolders.size > 0) { @@ -578,6 +717,7 @@ export class FaceclawVoiceControlBridge { private teardownCapture(): void { this.rawActive = false; this.streamProvider = null; + this.streamOptions = null; if (global.isAndroid) { this.controller?.stop(); } @@ -611,7 +751,7 @@ export class FaceclawVoiceControlBridge { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); // Push-to-talk on a capture of its own is not recorded. - if (this.captureHolders.has("continuous")) this.record(pcm); + if (this.captureHolders.has("continuous")) this.ensureRecording(); if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) { for (const listener of this.speechPauseListeners) listener(); } @@ -642,6 +782,10 @@ export class FaceclawVoiceControlBridge { ); } }, + onReplay: (kind: string, positionMs: number, durationMs: number) => { + const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) }; + for (const listener of Array.from(this.replayListeners)) listener(event); + }, }); this.controller.setListener(this.listenerProxy); } @@ -678,6 +822,21 @@ export class FaceclawVoiceControlBridge { } } +/** + * What a capture reads and transcribes with. A running capture is shared only + * by holders asking for the same: the provider and language, the microphones + * (the mixed stream, or the temples' own under a given configuration), or the + * recording it replays. + */ +function streamKey(options: PushToTalkOptions): string { + const source = options.replay + ? `replay:${options.replay.path}` + : options.micArray && !options.usePhoneMic + ? `array:${Array.from(options.micArray.configure).join(",")}` + : "mixed"; + return `${transcriptionKey(options)}|${source}`; +} + function toJavaFloats(values: Float32Array): any { const javaFloats = Array.create("float", values.length); for (let i = 0; i < values.length; i++) { @@ -687,3 +846,136 @@ function toJavaFloats(values: Float32Array): any { } export const voiceControlBridge = new FaceclawVoiceControlBridge(); + +export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed"; + +/** + * A recording played back as a capture of its own: transcribed by the same + * pipeline as the microphone, with the same settings, on a bridge and native + * controller separate from the microphone's. Nothing here touches the mic, + * so voice input and a live Transcribe window carry on beside a replay. + */ +export class VoiceReplay { + private readonly bridge = new FaceclawVoiceControlBridge(); + private readonly listeners = new Set<(jumpedToMs?: number) => void>(); + private readonly offReplay: () => void; + private position = 0; + private duration = 0; + private current: VoiceReplayState = "opening"; + private closed = false; + + /** + * `options` gives the capture options (provider, keys, processing) each + * time the replay starts, so a restart picks up changed settings. + */ + constructor(readonly path: string, private readonly options: () => PushToTalkOptions) { + this.offReplay = this.bridge.onReplay((event) => this.handle(event)); + } + + /** The replay's status, transcript and level events. */ + get events(): VoiceEventSource { + return this.bridge; + } + + /** Milliseconds into the recording. */ + get positionMs(): number { + return this.position; + } + + /** The recording's length in ms; 0 until it has been opened. */ + get durationMs(): number { + return this.duration; + } + + get state(): VoiceReplayState { + return this.current; + } + + /** + * The position, duration or state changed. `jumpedToMs` is given when + * playback moved there other than by playing on: a jump, or a start + * part-way in. It comes after the transcript of what played before. + */ + onChange(listener: (jumpedToMs?: number) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + /** Play from `positionMs` into the recording. */ + start(positionMs = 0): void { + if (this.closed) return; + this.position = Math.max(0, positionMs); + this.set("opening"); + this.bridge.startContinuousCapture({ + ...this.options(), + communicator: null, + usePhoneMic: false, + micArray: null, + replay: { path: this.path, startMs: this.position }, + }); + } + + /** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */ + jumpTo(positionMs: number): void { + if (this.closed) return; + const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs); + if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) { + this.bridge.seekReplay(target); + this.position = target; + this.notify(); + return; + } + this.start(target); + } + + setPaused(paused: boolean): void { + if (this.closed || (this.current !== "playing" && this.current !== "paused")) return; + this.bridge.setReplayPaused(paused); + this.set(paused ? "paused" : "playing"); + } + + /** Stop for good and release the capture. */ + close(): void { + if (this.closed) return; + this.closed = true; + this.offReplay(); + this.listeners.clear(); + this.bridge.stop(); + } + + private handle(event: VoiceReplayEvent): void { + this.duration = event.durationMs; + switch (event.kind) { + case "start": + this.position = event.positionMs; + this.current = "playing"; + this.notify(event.positionMs > 0 ? event.positionMs : undefined); + return; + case "seek": + this.position = event.positionMs; + this.notify(event.positionMs); + return; + case "end": + this.position = event.durationMs; + // Its last audio is still to be transcribed (a cloud provider's final). + this.bridge.finishContinuousCapture(); + this.set("ended"); + return; + case "error": + this.set("failed"); + return; + default: + this.position = event.positionMs; + this.notify(); + } + } + + private set(state: VoiceReplayState): void { + this.current = state; + this.notify(); + } + + private notify(jumpedToMs?: number): void { + for (const listener of Array.from(this.listeners)) listener(jumpedToMs); + } +} diff --git a/app/native/wav-header.ts b/app/native/wav-header.ts deleted file mode 100644 index a85ba2e0..00000000 --- a/app/native/wav-header.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */ -export function wavHeader(dataBytes: number): Uint8Array { - const header = new Uint8Array(44); - const view = new DataView(header.buffer); - const ascii = (offset: number, text: string) => { - for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i); - }; - ascii(0, "RIFF"); - view.setUint32(4, 36 + dataBytes, true); - ascii(8, "WAVE"); - ascii(12, "fmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); // PCM - view.setUint16(22, 1, true); // mono - view.setUint32(24, 16000, true); - view.setUint32(28, 32000, true); // byte rate - view.setUint16(32, 2, true); // block align - view.setUint16(34, 16, true); // bits per sample - ascii(36, "data"); - view.setUint32(40, dataBytes, true); - return header; -} diff --git a/app/native/wav-recorder.ios.ts b/app/native/wav-recorder.ios.ts deleted file mode 100644 index ff5e815a..00000000 --- a/app/native/wav-recorder.ios.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { knownFolders, path as paths } from '@nativescript/core' -import { wavHeader } from './wav-header' - -// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet. -const FLUSH_BYTES = 32000 - -/** - * Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is - * captured. The header's sizes are rewritten on every flush, so a crash leaves a - * playable file of everything up to the last second. - */ -export class WavRecorder { - private pending: Uint8Array[] = [] - private pendingBytes = 0 - private dataBytes = 0 - - private constructor(private handle: NSFileHandle | null, readonly path: string) {} - - /** A recorder for Downloads/.wav (a -2, -3... suffix if taken), or null. */ - static open(name: string): WavRecorder | null { - const fm = NSFileManager.defaultManager - const dir = paths.join(knownFolders.documents().path, 'Downloads') - try { - fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null) - let path = paths.join(dir, `${name}.wav`) - for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`) - if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null - const handle = NSFileHandle.fileHandleForWritingAtPath(path) - if (!handle || !handle.seekToEndReturningOffsetError(null)) return null - return new WavRecorder(handle, path) - } catch { - return null - } - } - - append(pcm: Uint8Array): void { - if (!this.handle || !pcm.length) return - this.pending.push(pcm.slice()) - this.pendingBytes += pcm.length - if (this.pendingBytes >= FLUSH_BYTES) this.flush() - } - - close(): void { - this.flush() - try { this.handle?.closeAndReturnError() } catch { /* already closed */ } - this.handle = null - } - - private flush(): void { - const handle = this.handle - if (!handle || !this.pendingBytes) return - const chunk = new Uint8Array(this.pendingBytes) - let offset = 0 - for (const part of this.pending) { chunk.set(part, offset); offset += part.length } - this.pending = [] - this.pendingBytes = 0 - try { - // NativeScript throws a failed call's NSError (storage full, file removed). - handle.writeDataError(toData(chunk)) - handle.seekToOffsetError(0) - handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length))) - handle.seekToEndReturningOffsetError(null) - this.dataBytes += chunk.length - return - } catch { - // Stop recording rather than retry. - } - try { handle.closeAndReturnError() } catch { /* ignore */ } - this.handle = null - } -} - -function toData(bytes: Uint8Array): NSData { - return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength) -} diff --git a/app/native/wav-recorder.ts b/app/native/wav-recorder.ts deleted file mode 100644 index cec96f3f..00000000 --- a/app/native/wav-recorder.ts +++ /dev/null @@ -1,42 +0,0 @@ -declare const android: any; -declare const java: any; -declare const com: any; - -const SAMPLE_RATE = 16000; - -/** - * Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory - * while it is captured (Android; wav-recorder.ios.ts is the iOS one). The - * native recorder keeps the header's sizes current, so a crash leaves a - * playable file of everything up to the last second. - */ -export class WavRecorder { - private constructor(private recorder: any, readonly path: string) {} - - /** A recorder for Downloads/.wav (a -2, -3... suffix if taken), or null. */ - static open(name: string): WavRecorder | null { - try { - const downloads = android.os.Environment.getExternalStoragePublicDirectory( - android.os.Environment.DIRECTORY_DOWNLOADS, - ); - downloads.mkdirs(); - let file = new java.io.File(downloads, `${name}.wav`); - for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`); - const path = String(file.getAbsolutePath()); - return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path); - } catch (error) { - console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`); - return null; - } - } - - /** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */ - append(pcm: any): void { - this.recorder?.append(pcm); - } - - close(): void { - this.recorder?.finish(); - this.recorder = null; - } -} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index d47a981b..bec7f6e5 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({ storageKey: "developer.useMicControl", defaultValue: true, description: - "Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.", + "Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.", }); export const showBleBandwidthSetting = new ConfigSettingBoolean({ @@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({ label: "Save Recording", storageKey: "voice.saveRecording", defaultValue: false, - description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.", + description: global.isIOS + ? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them." + : "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.", }); export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ diff --git a/app/ui/layers.ts b/app/ui/layers.ts index f41f3acf..90eb70de 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size"; import { dimPlanes, type Plane } from "../graphics/plane"; import { spanCurrent } from "../native/frame-timings"; import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings"; +import { type VoiceReplay } from "../native/voice-control"; import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures"; import { toViewport, type Viewport, type ViewportSize } from "./viewport"; @@ -13,8 +14,17 @@ export type TextSettingsEditToggle = { /** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */ export type ContinuousCaptureOptions = { - /** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */ + /** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */ recordAs?: string; + /** + * How many of the glasses' microphones to capture: 1 is their own mixed + * stream; 2 (one per temple) and 4 (both of each temple) stream the + * microphones separately through the custom firmware, for recording and + * array processing. While any client asks for more than one, every capture + * uses the most asked for. Glasses without the microphone channel, and the + * phone's microphone, give 1 whatever is asked. + */ + mics?: MicCount; /** * Transcribe with this provider, expecting this spoken language (ISO 639-1, * or "" for the provider to detect it), instead of the Transcription @@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = { export type TranscriptionChoice = { provider: VoiceProvider; language: string }; +export type MicCount = 1 | 2 | 4; + export type LayerActions = { /** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */ requestRender: () => void; @@ -55,6 +67,12 @@ export type LayerActions = { */ startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise | void; stopContinuousVoiceCapture: (client: string) => Promise | void; + /** + * Open a recording (.mka) for playing back through transcription in place + * of the microphone; the caller starts and closes it. Absent, or null, + * where recordings can't be replayed. + */ + openVoiceReplay?: (path: string) => VoiceReplay | null; /** Play a CFW tone-sequencer payload (see sound-effects.ts). */ playBuzzerSequence: (payload: Uint8Array) => Promise | void; }; diff --git a/app/ui/listening-header.ts b/app/ui/listening-header.ts index 4c334d19..762fcaaa 100644 --- a/app/ui/listening-header.ts +++ b/app/ui/listening-header.ts @@ -1,6 +1,6 @@ import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image"; import { truncateText } from "../graphics/textwrap"; -import { voiceControlBridge } from "../native/voice-control"; +import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control"; import { lineStep } from "./metrics"; import type { Edges } from "./viewport"; @@ -68,8 +68,9 @@ export class MicLevelMeter { private timer: ReturnType | null = null; private unsubscribe: (() => void) | null = null; - start(requestRender: () => void, isVisible: () => boolean): void { - this.unsubscribe = voiceControlBridge.onMicLevel((level) => { + /** `source` is the capture whose level to show: the microphone's unless given. */ + start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void { + this.unsubscribe = source.onMicLevel((level) => { this.peak = Math.max(this.peak, level); }); this.timer = setInterval(() => { diff --git a/app/ui/shell/in-process-window.ts b/app/ui/shell/in-process-window.ts index d0b7990c..7061d509 100644 --- a/app/ui/shell/in-process-window.ts +++ b/app/ui/shell/in-process-window.ts @@ -48,6 +48,8 @@ export type InProcessWindowOptions = { setScreenOn?: ShellWindow["setScreenOn"]; /** See ShellWindow.keepsScreenOn. */ keepsScreenOn?: ShellWindow["keepsScreenOn"]; + /** See ShellWindow.restoresApp. */ + restoresApp?: boolean; /** The window came to or left the foreground (after its surface visibility follows). */ onForegroundChanged?: (foreground: boolean) => void; /** Shared actions; requestRender is rebound to this window's render. */ @@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces holdToTalk: options.holdToTalk, isVoiceCapturing: options.isVoiceCapturing, keepsScreenOn: options.keepsScreenOn, + restoresApp: options.restoresApp, setScreenOn: options.setScreenOn, heightMode, hasAppMenu: () => appMenuItems().length > 0, diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 683bc97c..94c743ca 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -151,6 +151,13 @@ export type ShellWindow = { * draws this window's top bar. */ heightMode: WindowHeightMode; + /** + * False for a window that must not bring its app back when Faceclaw + * restarts (the controller's open-apps restore relaunches apps, not + * windows): one that shows something a plain launch of its app would not, + * where the relaunch would do something else in its place. + */ + restoresApp?: boolean; /** App-side cleanup when the shell closes the window (worker notification, surface removal). */ close?: () => void; drawIcon: ShellChromeWindow["drawIcon"]; diff --git a/native/kotlin/README.md b/native/kotlin/README.md index 4ad08a49..40e73b9e 100644 --- a/native/kotlin/README.md +++ b/native/kotlin/README.md @@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore callback interfaces. Since 2026-09-22 also the platform-neutral cores behind the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`, - `audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader), + `audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska + writer/reader and the MicRecording capture format built on it, MicArrayProtocol), `graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt, ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream, ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`, diff --git a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt index 7e1e63f9..ff0bcdaf 100644 --- a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt +++ b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt @@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis actual fun deleteFile(path: String): Boolean = java.io.File(path).delete() actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to)) + +actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput { + private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) } + override var size = 0L + private set + + override fun write(bytes: ByteArray, offset: Int, length: Int) { + file.write(bytes, offset, length) + size += length + } + + override fun writeAt(position: Long, bytes: ByteArray) { + require(position >= 0 && position + bytes.size <= size) + file.seek(position) + file.write(bytes) + file.seek(size) + } + + override fun close() = file.close() +} + +actual fun openFileInput(path: String): SeekableInput = object : SeekableInput { + private val file = java.io.RandomAccessFile(path, "r") + override val length = file.length() + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int { + file.seek(position) + var total = 0 + while (total < length) { + val read = file.read(buffer, offset + total, length - total) + if (read <= 0) break + total += read + } + return total + } + + override fun close() = file.close() +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt index 739c23e9..e1c717f7 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt @@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long expect fun deleteFile(path: String): Boolean expect fun renameFile(from: String, to: String): Boolean + +/** A file being written front to back whose earlier bytes can still be patched (container headers). */ +interface SeekableOutput { + /** Bytes appended so far. */ + val size: Long + + /** Appends [length] bytes of [bytes] from [offset]. */ + fun write(bytes: ByteArray, offset: Int, length: Int) + + /** Overwrites bytes already written, at [position]; later appends still go to the end. */ + fun writeAt(position: Long, bytes: ByteArray) + + fun close() +} + +/** A file read at arbitrary positions without loading it whole. */ +interface SeekableInput { + val length: Long + + /** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */ + fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int + + fun close() +} + +/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */ +expect fun openFileOutput(path: String): SeekableOutput + +/** Opens [path] for reading. Throws when it cannot. */ +expect fun openFileInput(path: String): SeekableInput diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Matroska.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Matroska.kt new file mode 100644 index 00000000..897b700c --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Matroska.kt @@ -0,0 +1,764 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField + +/** + * A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that + * each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and + * file attachments. Every block carries its own timestamp, so tracks on independent clocks and + * gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it. + * + * The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of + * about a second each, cues and the segment size written when the recording finishes. A + * recording cut short by a crash is still a valid live-style stream of everything up to its last + * complete cluster, and the reader takes it as such. + */ +internal object MatroskaIds { + const val EBML = 0x1A45DFA3 + const val EBML_VERSION = 0x4286 + const val EBML_READ_VERSION = 0x42F7 + const val EBML_MAX_ID_LENGTH = 0x42F2 + const val EBML_MAX_SIZE_LENGTH = 0x42F3 + const val DOC_TYPE = 0x4282 + const val DOC_TYPE_VERSION = 0x4287 + const val DOC_TYPE_READ_VERSION = 0x4285 + const val VOID = 0xEC + const val SEGMENT = 0x18538067 + const val SEEK_HEAD = 0x114D9B74 + const val SEEK = 0x4DBB + const val SEEK_ID = 0x53AB + const val SEEK_POSITION = 0x53AC + const val INFO = 0x1549A966 + const val TIMESTAMP_SCALE = 0x2AD7B1 + const val DURATION = 0x4489 + const val DATE_UTC = 0x4461 + const val TITLE = 0x7BA9 + const val MUXING_APP = 0x4D80 + const val WRITING_APP = 0x5741 + const val TRACKS = 0x1654AE6B + const val TRACK_ENTRY = 0xAE + const val TRACK_NUMBER = 0xD7 + const val TRACK_UID = 0x73C5 + const val TRACK_TYPE = 0x83 + const val FLAG_DEFAULT = 0x88 + const val FLAG_LACING = 0x9C + const val NAME = 0x536E + const val LANGUAGE = 0x22B59C + const val CODEC_ID = 0x86 + const val AUDIO = 0xE1 + const val SAMPLING_FREQUENCY = 0xB5 + const val CHANNELS = 0x9F + const val BIT_DEPTH = 0x6264 + const val ATTACHMENTS = 0x1941A469 + const val ATTACHED_FILE = 0x61A7 + const val FILE_NAME = 0x466E + const val FILE_MIME_TYPE = 0x4660 + const val FILE_DATA = 0x465C + const val FILE_UID = 0x46AE + const val CLUSTER = 0x1F43B675 + const val TIMESTAMP = 0xE7 + const val SIMPLE_BLOCK = 0xA3 + const val BLOCK_GROUP = 0xA0 + const val BLOCK = 0xA1 + const val BLOCK_DURATION = 0x9B + const val CUES = 0x1C53BB6B + const val CUE_POINT = 0xBB + const val CUE_TIME = 0xB3 + const val CUE_TRACK_POSITIONS = 0xB7 + const val CUE_TRACK = 0xF7 + const val CUE_CLUSTER_POSITION = 0xF1 +} + +/** Builds EBML elements into a byte buffer. */ +internal class EbmlWriter(capacity: Int = 256) { + private val sink = ByteSink(capacity) + + fun size(): Int = sink.size() + + fun bytes(): ByteArray = sink.toByteArray() + + /** An element ID, written as given: its length marker is part of the value. */ + fun id(id: Int) { + if (id > 0xFFFFFF) sink.write(id ushr 24) + if (id > 0xFFFF) sink.write(id ushr 16) + if (id > 0xFF) sink.write(id ushr 8) + sink.write(id) + } + + /** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */ + fun dataSize(size: Long, width: Int = 0) { + var bytes = width + if (bytes == 0) { + bytes = 1 + // The all-ones value of each width means "unknown size". + while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++ + } + val marked = size or (1L shl (7 * bytes)) + for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt()) + } + + fun uint(id: Int, value: Long) { + var bytes = 1 + while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++ + id(id) + dataSize(bytes.toLong()) + for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt()) + } + + /** A fixed eight-byte integer (dates). */ + fun int64(id: Int, value: Long) { + id(id) + dataSize(8) + for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt()) + } + + fun float64(id: Int, value: Double) = int64(id, value.toRawBits()) + + fun string(id: Int, text: String) { + val bytes = text.encodeToByteArray() + binary(id, bytes, 0, bytes.size) + } + + fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) { + id(id) + dataSize(length.toLong()) + sink.write(bytes, offset, length) + } + + fun raw(bytes: ByteArray) = sink.write(bytes) + + fun raw(value: Int) = sink.write(value) + + fun master(id: Int, build: EbmlWriter.() -> Unit) { + val child = EbmlWriter() + child.build() + id(id) + dataSize(child.size().toLong()) + sink.write(child.bytes()) + } + + companion object { + /** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */ + fun padded(element: ByteArray, reserved: Int): ByteArray? { + val filler = reserved - element.size + // A Void element is at least two bytes. + if (filler < 0 || filler == 1) return null + val out = element.copyOf(reserved) + if (filler == 0) return out + val at = element.size + out[at] = MatroskaIds.VOID.toByte() + if (filler < 9) { + out[at + 1] = (0x80 or (filler - 2)).toByte() + } else { + val size = (filler - 9).toLong() + out[at + 1] = 0x01 + for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte() + } + return out + } + } +} + +class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray) + +/** + * Writes one Matroska audio file. Tracks can be added at any time before the first block that + * uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are + * milliseconds from the start of the file. Not thread-safe. + */ +class MkaWriter( + private val output: SeekableOutput, + title: String?, + /** When the recording began (Unix epoch ms), for the file's date. */ + dateUtcMs: Long, + attachments: List, +) { + companion object { + const val CODEC_PCM = "A_PCM/INT/LIT" + const val CODEC_TEXT = "S_TEXT/UTF8" + const val TRACK_AUDIO = 2 + const val TRACK_SUBTITLE = 0x11 + const val APP_NAME = "Faceclaw" + private const val SEEK_RESERVED = 128 + private const val TRACKS_RESERVED = 4096 + private const val CLUSTER_SPAN_MS = 1000L + private const val CLUSTER_MAX_BYTES = 1 shl 20 + // Block timestamps are signed 16-bit offsets from their cluster's. + private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L + private const val DURATION_PATCH_CLUSTERS = 5 + // Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC. + private const val DATE_EPOCH_MS = 978_307_200_000L + } + + private class Track( + val number: Int, + val type: Int, + val name: String, + val sampleRate: Int, + val channels: Int, + val preferred: Boolean, + ) + + private class Cue(val clusterPosition: Long, val timeMs: Long) + + private val tracks = ArrayList() + private val cues = ArrayList() + private val segmentSizePosition: Long + private val segmentStart: Long + private val seekPosition: Long + private val infoPosition: Long + private val durationPosition: Long + private val tracksPosition: Long + private val attachmentsPosition: Long + private var cluster: EbmlWriter? = null + private var clusterBaseMs = 0L + private var clustersSincePatch = 0 + private var finished = false + private val hasAttachments = attachments.isNotEmpty() + + /** The end of the latest audio or text written, in ms from the start of the file. */ + var durationMs = 0L + private set + + init { + val head = EbmlWriter() + head.master(MatroskaIds.EBML) { + uint(MatroskaIds.EBML_VERSION, 1) + uint(MatroskaIds.EBML_READ_VERSION, 1) + uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4) + uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8) + string(MatroskaIds.DOC_TYPE, "matroska") + uint(MatroskaIds.DOC_TYPE_VERSION, 4) + uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2) + } + head.id(MatroskaIds.SEGMENT) + segmentSizePosition = head.size().toLong() + // Unknown size (a live stream) until finish() knows it. + head.raw(0x01) + repeat(7) { head.raw(0xFF) } + segmentStart = head.size().toLong() + append(head.bytes()) + + seekPosition = output.size + infoPosition = seekPosition + SEEK_RESERVED + val info = EbmlWriter() + info.master(MatroskaIds.INFO) { + uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000) + string(MatroskaIds.MUXING_APP, APP_NAME) + string(MatroskaIds.WRITING_APP, APP_NAME) + int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000) + if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title) + // Last, so its eight bytes are the element's final ones. + float64(MatroskaIds.DURATION, 0.0) + } + durationPosition = infoPosition + info.size() - 8 + tracksPosition = infoPosition + info.size() + attachmentsPosition = tracksPosition + TRACKS_RESERVED + + append(seekHead(-1)) + append(info.bytes()) + append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!) + if (attachments.isNotEmpty()) { + val files = EbmlWriter() + files.master(MatroskaIds.ATTACHMENTS) { + attachments.forEachIndexed { index, file -> + master(MatroskaIds.ATTACHED_FILE) { + string(MatroskaIds.FILE_NAME, file.name) + string(MatroskaIds.FILE_MIME_TYPE, file.mimeType) + binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size) + uint(MatroskaIds.FILE_UID, index + 1L) + } + } + } + append(files.bytes()) + } + } + + private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size) + + private fun seekHead(cuesPosition: Long): ByteArray { + val head = EbmlWriter() + head.master(MatroskaIds.SEEK_HEAD) { + fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) { + val idBytes = EbmlWriter() + idBytes.id(id) + binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size()) + uint(MatroskaIds.SEEK_POSITION, position - segmentStart) + } + entry(MatroskaIds.INFO, infoPosition) + entry(MatroskaIds.TRACKS, tracksPosition) + if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition) + if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition) + } + return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space") + } + + /** + * Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list + * is full. A [preferred] track is listed ahead of the others, and the first listed track is + * the one players pick by default. + */ + fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int = + addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred)) + + /** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */ + fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false)) + + private fun addTrack(track: Track): Int { + check(!finished) { "recording already finished" } + // A block's track number is written in one byte. + if (track.number > 126) return -1 + tracks.add(track) + val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED) + if (region == null) { + tracks.remove(track) + return -1 + } + output.writeAt(tracksPosition, region) + return track.number + } + + private fun listedTracks(): List = + tracks.filter { it.type == TRACK_AUDIO && it.preferred } + + tracks.filter { it.type == TRACK_AUDIO && !it.preferred } + + tracks.filter { it.type != TRACK_AUDIO } + + private fun tracksElement(): ByteArray { + val listed = listedTracks() + val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO } + val out = EbmlWriter(1024) + out.master(MatroskaIds.TRACKS) { + for (track in listed) { + master(MatroskaIds.TRACK_ENTRY) { + uint(MatroskaIds.TRACK_NUMBER, track.number.toLong()) + uint(MatroskaIds.TRACK_UID, track.number.toLong()) + uint(MatroskaIds.TRACK_TYPE, track.type.toLong()) + uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0) + uint(MatroskaIds.FLAG_LACING, 0) + string(MatroskaIds.NAME, track.name) + string(MatroskaIds.LANGUAGE, "und") + if (track.type == TRACK_AUDIO) { + string(MatroskaIds.CODEC_ID, CODEC_PCM) + master(MatroskaIds.AUDIO) { + float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble()) + uint(MatroskaIds.CHANNELS, track.channels.toLong()) + uint(MatroskaIds.BIT_DEPTH, 16) + } + } else { + string(MatroskaIds.CODEC_ID, CODEC_TEXT) + } + } + } + } + return out.bytes() + } + + private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number") + + /** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */ + fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) { + val track = track(trackNumber) + require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" } + if (length <= 0) return + val block = openCluster(timeMs) + block.id(MatroskaIds.SIMPLE_BLOCK) + block.dataSize(4L + length) + blockHeader(block, trackNumber, timeMs, 0x80) + block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length)) + val frames = length / (2 * track.channels) + noteEnd(timeMs + frames * 1000L / track.sampleRate) + } + + /** Appends one string to a text track, shown from [timeMs] for [durationMs]. */ + fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) { + require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" } + val bytes = text.encodeToByteArray() + val block = openCluster(timeMs) + block.master(MatroskaIds.BLOCK_GROUP) { + id(MatroskaIds.BLOCK) + dataSize(4L + bytes.size) + blockHeader(this, trackNumber, timeMs, 0) + raw(bytes) + uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs)) + } + noteEnd(timeMs + maxOf(1L, durationMs)) + } + + private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) { + val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt() + block.raw(0x80 or trackNumber) + block.raw(relative shr 8) + block.raw(relative) + block.raw(flags) + } + + private fun noteEnd(endMs: Long) { + if (endMs > durationMs) durationMs = endMs + } + + /** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */ + private fun openCluster(timeMs: Long): EbmlWriter { + check(!finished) { "recording already finished" } + val time = maxOf(0L, timeMs) + val open = cluster + if (open != null) { + val relative = time - clusterBaseMs + if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open + flushCluster() + } + clusterBaseMs = time + return EbmlWriter(64 * 1024).also { cluster = it } + } + + private fun flushCluster() { + val body = cluster ?: return + cluster = null + if (body.size() == 0) return + val timestamp = EbmlWriter(16) + timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs) + val header = EbmlWriter(32) + header.id(MatroskaIds.CLUSTER) + header.dataSize(timestamp.size().toLong() + body.size()) + header.raw(timestamp.bytes()) + cues.add(Cue(output.size - segmentStart, clusterBaseMs)) + append(header.bytes()) + append(body.bytes()) + if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration() + } + + private fun patchDuration() { + clustersSincePatch = 0 + val value = durationMs.toDouble().toRawBits() + output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() }) + } + + /** Writes what is buffered, then the cues, duration and segment size, and closes the file. */ + fun finish() { + if (finished) return + try { + flushCluster() + finished = true + val cuesPosition = output.size + val cueTrack = listedTracks().firstOrNull()?.number ?: 1 + if (cues.isNotEmpty()) { + val index = EbmlWriter(cues.size * 16 + 16) + index.master(MatroskaIds.CUES) { + for (cue in cues) { + master(MatroskaIds.CUE_POINT) { + uint(MatroskaIds.CUE_TIME, cue.timeMs) + master(MatroskaIds.CUE_TRACK_POSITIONS) { + uint(MatroskaIds.CUE_TRACK, cueTrack.toLong()) + uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition) + } + } + } + } + append(index.bytes()) + output.writeAt(seekPosition, seekHead(cuesPosition)) + } + patchDuration() + val size = EbmlWriter(8) + size.dataSize(output.size - segmentStart, 8) + output.writeAt(segmentSizePosition, size.bytes()) + } finally { + finished = true + output.close() + } + } +} + +class MkaTrackInfo( + @JvmField val number: Int, + /** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */ + @JvmField val type: Int, + @JvmField val codecId: String, + @JvmField val name: String, + @JvmField val sampleRate: Int, + @JvmField val channels: Int, + @JvmField val bitDepth: Int, +) { + val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16 +} + +class MkaBlock( + @JvmField val track: Int, + @JvmField val timeMs: Long, + @JvmField val data: ByteArray, + /** From the block's group, or -1 when the file doesn't say (audio blocks). */ + @JvmField val durationMs: Long, +) + +/** + * Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what + * [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws + * IllegalArgumentException for anything that is not such a file. + */ +class MkaReader(private val input: SeekableInput) { + private class ClusterRef(val position: Long, val size: Int, val rawTime: Long) + + private val clusters = ArrayList() + private val trackList = ArrayList() + private val attachmentList = ArrayList() + private var timestampScale = 1_000_000L + private var infoDurationMs = 0L + private var nextCluster = 0 + private var blocks: List = emptyList() + private var blockIndex = 0 + private var skipBeforeMs = 0L + + val tracks: List get() = trackList + val attachments: List get() = attachmentList + + /** The end of the last audio or text in the file, in ms. */ + val durationMs: Long + + init { + scan() + var end = infoDurationMs + if (clusters.isNotEmpty()) { + for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block)) + } + durationMs = end + } + + fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number } + + /** Continue from the first block at or after [timeMs]. */ + fun seekTo(timeMs: Long) { + val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs } + nextCluster = maxOf(0, index) + blocks = emptyList() + blockIndex = 0 + skipBeforeMs = timeMs + } + + /** The next block, or null at the end of the file. */ + fun next(): MkaBlock? { + while (true) { + while (blockIndex >= blocks.size) { + if (nextCluster >= clusters.size) return null + blocks = readCluster(clusters[nextCluster++]) + blockIndex = 0 + } + val block = blocks[blockIndex++] + if (block.timeMs >= skipBeforeMs) return block + } + } + + fun close() = input.close() + + private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L + + private fun blockSpanMs(block: MkaBlock): Long { + if (block.durationMs >= 0) return block.durationMs + val track = track(block.track) ?: return 0 + if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0 + return block.data.size / (2L * track.channels) * 1000L / track.sampleRate + } + + /** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */ + private class Header(val id: Int, val size: Long, val length: Int) + + private fun header(bytes: ByteArray, at: Int, end: Int): Header? { + if (at >= end) return null + val first = bytes[at].toInt() and 0xff + val idLength = first.countLeadingZeroBits() - 24 + 1 + if (idLength > 4 || at + idLength >= end) return null + var id = 0 + for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff) + val sizeAt = at + idLength + val sizeFirst = bytes[sizeAt].toInt() and 0xff + val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1 + if (sizeLength > 8 || sizeAt + sizeLength > end) return null + var size = (sizeFirst and (0xff shr sizeLength)).toLong() + var allOnes = size == (0xff shr sizeLength).toLong() + for (index in 1 until sizeLength) { + val byte = bytes[sizeAt + index].toInt() and 0xff + if (byte != 0xff) allOnes = false + size = (size shl 8) or byte.toLong() + } + return Header(id, if (allOnes) -1 else size, idLength + sizeLength) + } + + private fun readAt(position: Long, length: Int): ByteArray { + val buffer = ByteArray(length) + val read = input.readAt(position, buffer, 0, length) + return if (read == length) buffer else buffer.copyOf(maxOf(0, read)) + } + + private fun uint(bytes: ByteArray, at: Int, size: Int): Long { + var value = 0L + for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff) + return value + } + + private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) { + 4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble() + 8 -> Double.fromBits(uint(bytes, at, 8)) + else -> 0.0 + } + + /** Calls [visit] with the ID, payload offset and payload size of each complete child element. */ + private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) { + var at = start + while (true) { + val child = header(bytes, at, end) ?: return + val payload = at + child.length + if (child.size < 0 || child.size > end - payload) return + visit(child.id, payload, child.size.toInt()) + at = payload + child.size.toInt() + } + } + + private fun scan() { + val length = input.length + val start = readAt(0, 64) + val ebml = header(start, 0, start.size) + require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" } + var position = ebml.length + ebml.size + val segment = readAt(position, 12).let { header(it, 0, it.size) } + require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" } + position += segment.length + val end = if (segment.size < 0) length else minOf(length, position + segment.size) + while (position < end) { + val head = readAt(position, 12) + val element = header(head, 0, head.size) ?: break + val payload = position + element.length + if (element.id == MatroskaIds.CLUSTER) { + require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" } + // A recording cut short ends in a partial cluster; its complete blocks still count. + val available = minOf(element.size, end - payload) + require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" } + clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt()))) + } else if (element.size < 0) { + break + } else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) { + when (element.id) { + MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt())) + MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt())) + MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt())) + } + } + if (element.size < 0) break + position = payload + element.size + } + } + + private fun clusterTime(position: Long, size: Int): Long { + val head = readAt(position, minOf(size, 16)) + val first = header(head, 0, head.size) + if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) { + return uint(head, first.length, first.size.toInt()) + } + val whole = readAt(position, size) + var time = 0L + children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) } + return time + } + + private fun parseInfo(bytes: ByteArray) { + var duration = 0.0 + children(bytes, 0, bytes.size) { id, at, size -> + when (id) { + MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size)) + MatroskaIds.DURATION -> duration = float(bytes, at, size) + } + } + infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong() + } + + private fun parseTracks(bytes: ByteArray) { + trackList.clear() + children(bytes, 0, bytes.size) { id, at, size -> + if (id == MatroskaIds.TRACK_ENTRY) { + var number = 0 + var type = 0 + var codec = "" + var name = "" + var rate = 8000.0 + var channels = 1 + var depth = 0 + children(bytes, at, at + size) { field, fieldAt, fieldSize -> + when (field) { + MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt() + MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt() + MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000') + MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000') + MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize -> + when (audio) { + MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize) + MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt() + MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt() + } + } + } + } + val audio = type == MkaWriter.TRACK_AUDIO + trackList.add( + MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0), + ) + } + } + } + + private fun parseAttachments(bytes: ByteArray) { + children(bytes, 0, bytes.size) { id, at, size -> + if (id == MatroskaIds.ATTACHED_FILE) { + var name = "" + var mime = "" + var data = ByteArray(0) + children(bytes, at, at + size) { field, fieldAt, fieldSize -> + when (field) { + MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize) + MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize) + MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize) + } + } + attachmentList.add(MkaAttachment(name, mime, data)) + } + } + } + + private fun readCluster(cluster: ClusterRef): List { + val bytes = readAt(cluster.position, cluster.size) + val out = ArrayList() + var clusterRaw = cluster.rawTime + children(bytes, 0, bytes.size) { id, at, size -> + when (id) { + MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size) + MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) } + MatroskaIds.BLOCK_GROUP -> { + var blockAt = -1 + var blockSize = 0 + var duration = -1L + children(bytes, at, at + size) { field, fieldAt, fieldSize -> + when (field) { + MatroskaIds.BLOCK -> { + blockAt = fieldAt + blockSize = fieldSize + } + MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize)) + } + } + if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) } + } + } + } + return out + } + + /** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */ + private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? { + if (size < 4) return null + val first = bytes[at].toInt() and 0xff + val trackLength = first.countLeadingZeroBits() - 24 + 1 + if (trackLength > 2 || size < trackLength + 3) return null + var track = first and (0xff shr trackLength) + if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff) + val head = at + trackLength + val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong() + val flags = bytes[head + 2].toInt() and 0xff + if (flags and 0x06 != 0) return null + return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicArrayProtocol.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicArrayProtocol.kt new file mode 100644 index 00000000..2014532b --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicArrayProtocol.kt @@ -0,0 +1,93 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic + +/** + * The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM' + * multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The + * CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the + * contract is the header of g2flash patches/mic_control.c). + */ +object MicArrayProtocol { + const val VERSION = 1 + const val HEADER_BYTES = 21 + const val OP_CONFIGURE = 1 + const val OP_STOP = 3 + const val OP_RENEW = 4 + /** How long an armed temple streams without a CONFIGURE or RENEW. */ + const val LEASE_MS = 90_000L + const val FLAG_TRUNCATED = 0x80 + const val CODEC_RAW = 1 + const val FORMAT_PCM16 = 0 + + /** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */ + class Frame( + /** Low 16 bits of the temple's frame counter. */ + @JvmField val sequence: Int, + /** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */ + @JvmField val deviceTickMs: Long, + @JvmField val channels: Int, + @JvmField val sampleRate: Int, + @JvmField val format: Int, + @JvmField val codec: Int, + /** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */ + @JvmField val angleDegrees: Int, + @JvmField val ssr: Int, + /** The firmware cut the payload short of the samples it captured. */ + @JvmField val truncated: Boolean, + @JvmField val payloadOffset: Int, + @JvmField val payloadLength: Int, + ) + + /** A control record with no payload: ['M','C', version, op]. */ + @JvmStatic + fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte()) + + @JvmStatic + fun isConfigure(record: ByteArray?): Boolean = + record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() && + record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte() + + /** Decodes a stream frame's header, or returns null when [data] isn't one. */ + @JvmStatic + fun decodeFrame(data: ByteArray): Frame? { + if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) { + return null + } + val length = minOf(u16(data, 19), data.size - HEADER_BYTES) + return Frame( + sequence = u16(data, 4), + deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16), + channels = data[10].toInt() and 0xff, + sampleRate = u16(data, 11) * 100, + format = data[13].toInt() and 0xff, + codec = data[14].toInt() and 0xff, + angleDegrees = u16(data, 15).toShort().toInt(), + ssr = u16(data, 17).toShort().toInt(), + truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0, + payloadOffset = HEADER_BYTES, + payloadLength = length, + ) + } + + /** + * The frame's samples as interleaved PCM16 (whole sample frames only), or null when the + * payload isn't raw 16-bit PCM or holds none. + */ + @JvmStatic + fun pcm16(data: ByteArray, frame: Frame): ShortArray? { + if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null + val frames = frame.payloadLength / (2 * frame.channels) + if (frames <= 0) return null + val samples = ShortArray(frames * frame.channels) + var at = frame.payloadOffset + for (index in samples.indices) { + samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1]) + at += 2 + } + return samples + } + + private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8) +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicRecording.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicRecording.kt new file mode 100644 index 00000000..4488b007 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/MicRecording.kt @@ -0,0 +1,484 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic +import kotlin.math.abs + +/** Which microphones a chunk of capture audio came from. */ +object MicStream { + /** One mixed channel: the glasses' stock stream, or the phone's own microphone. */ + const val MONO = 0 + /** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */ + const val LEFT = 1 + const val RIGHT = 2 + /** The mono feed derived from the temples for speech recognition: recorded, never replayed. */ + const val MIX = 3 + const val COUNT = 4 + + /** The stream's name in frame metadata. */ + @JvmStatic + fun key(stream: Int): String = when (stream) { + MONO -> "mic" + LEFT -> "left" + RIGHT -> "right" + else -> "mix" + } + + /** The name of the stream's audio tracks in a recording. */ + @JvmStatic + fun trackName(stream: Int): String = when (stream) { + MONO -> "mic" + LEFT -> "left temple" + RIGHT -> "right temple" + else -> "mix" + } + + /** The stream a recording's track or metadata names, or -1. */ + @JvmStatic + fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1 +} + +/** + * One chunk of microphone audio as a capture source delivered it (about 50 ms), before any + * processing. Live captures, recordings and replays all trade in these. + */ +class MicChunk( + @JvmField val stream: Int, + @JvmField val channels: Int, + @JvmField val sampleRate: Int, + /** Interleaved PCM16; the first [frames] sample frames are the chunk. */ + @JvmField val pcm: ShortArray, + @JvmField val frames: Int, + /** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */ + @JvmField val timeMs: Long, +) { + /** Whether the source reported a direction of arrival with the chunk. */ + @JvmField var hasDirection = false + @JvmField var angleDegrees = 0 + @JvmField var ssr = 0 + /** The source's packet counter, or -1 when it has none. */ + @JvmField var sequence = -1 + /** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */ + @JvmField var deviceTickMs = -1L + /** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */ + @JvmField var clockStream = stream + /** The source cut the chunk short of what it captured. */ + @JvmField var truncated = false + /** Sample frames known to be missing between the previous chunk of this stream and this one. */ + @JvmField var lostFramesBefore = 0 + + val durationMs: Long get() = frames * 1000L / sampleRate +} + +/** + * Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for + * array processing (beamforming, noise cancellation): replays feed it the recorded chunks with + * the times they first arrived at, so an implementation that decides from the chunks alone + * behaves the same on a replay as it did live. + */ +interface MicArrayMixer { + /** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */ + fun mix(chunk: MicChunk): MicChunk? + + fun reset() +} + +/** + * The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix). + * The temples sample on independent clocks, so their audio is never summed. The left temple + * feeds whenever it is streaming; the right one stands in while it is not. + */ +class SingleTempleMixer : MicArrayMixer { + companion object { + /** A temple that has delivered nothing for this long no longer counts as streaming. */ + const val SILENT_MS = 300L + } + + private var leftAtMs = Long.MIN_VALUE + + override fun reset() { + leftAtMs = Long.MIN_VALUE + } + + override fun mix(chunk: MicChunk): MicChunk? { + if (chunk.stream == MicStream.LEFT) { + leftAtMs = chunk.timeMs + } else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) { + return null + } + val channels = chunk.channels + val mono = ShortArray(chunk.frames) + for (frame in 0 until chunk.frames) { + var sum = 0 + for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel] + mono[frame] = (sum / channels).toShort() + } + val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs) + out.hasDirection = chunk.hasDirection + out.angleDegrees = chunk.angleDegrees + out.ssr = chunk.ssr + out.deviceTickMs = chunk.deviceTickMs + out.clockStream = chunk.stream + out.truncated = chunk.truncated + return out + } +} + +/** + * Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and + * layout as each first appears, plus a "frames" text track with one JSON object per chunk that + * has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md. + * + * Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick, + * or the count of samples from a source without one) says how far apart its chunks were + * captured; the least-delayed recent arrival places that clock on the file's timeline. So a + * chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap. + * + * Safe to share between the capture thread and whoever finishes the recording. + */ +class MicRecordingWriter( + output: SeekableOutput, + /** A JSON document describing the capture, stored as the file's "capture.json" attachment. */ + configJson: String?, + /** When the recording began (Unix epoch ms). */ + startedAtMs: Long, + platform: ProtocolPlatform = protocolPlatform(), + private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) }, +) { + companion object { + const val CONFIG_ATTACHMENT = "capture.json" + const val META_TRACK = "frames" + /** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */ + const val DISCONTINUITY_MS = 5000L + private const val WINDOW_BUCKETS = 8 + private const val BUCKET_MS = 1000L + } + + /** + * Places one track's chunks on the file's timeline. `source` time is the stream's own clock + * since this clock was anchored; the offset from it to arrival time is taken as the least + * seen over the last few seconds, which is the least-delayed delivery. + */ + private class TrackClock(val number: Int, val sampleRate: Int) { + private var anchored = false + private var clockStream = -1 + private var tickBase = -1L + private var lastTick = -1L + private var frames = 0L + private val minima = LongArray(WINDOW_BUCKETS) + private var bucket = 0 + private var bucketStartMs = 0L + var endMs = Long.MIN_VALUE + private set + + fun reset() { + anchored = false + } + + private fun offset(): Long = minima.min() + + fun stamp(chunk: MicChunk): Long { + val arrival = chunk.timeMs + val ticked = chunk.deviceTickMs >= 0 + if (anchored) { + // Another clock (the feed moved to the other temple), or this one restarted. + if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false + else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false + } + var anchorNow = !anchored + var source = 0L + if (anchored) { + frames += chunk.lostFramesBefore + source = sourceTime(chunk) + if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true + } + if (anchorNow) { + anchored = true + clockStream = chunk.clockStream + tickBase = if (ticked) chunk.deviceTickMs else -1L + frames = 0 + minima.fill(Long.MAX_VALUE) + bucket = 0 + bucketStartMs = arrival + source = 0 + } + if (arrival - bucketStartMs >= BUCKET_MS) { + bucket = (bucket + 1) % WINDOW_BUCKETS + minima[bucket] = Long.MAX_VALUE + bucketStartMs = arrival + } + if (arrival - source < minima[bucket]) minima[bucket] = arrival - source + // Never before the end of the track's previous block. + val time = maxOf(source + offset(), endMs) + frames += chunk.frames + lastTick = chunk.deviceTickMs + endMs = time + chunk.durationMs + return time + } + + private fun sourceTime(chunk: MicChunk): Long = + if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate + } + + private val lock = platform.createLock() + private val writer: MkaWriter + private val clocks = HashMap() + private var metaTrack = 0 + private var originMs = Long.MIN_VALUE + private var lastTimeMs = 0L + private var closed = false + + /** The recording stopped early: the file could not be written. */ + @Volatile + var failed = false + private set + + init { + val attachments = if (configJson.isNullOrEmpty()) emptyList() + else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray())) + writer = MkaWriter(output, null, startedAtMs, attachments) + } + + /** Length of the recording so far, in ms. */ + val durationMs: Long get() = lock.withLock { writer.durationMs } + + /** Appends [chunk] to its stream's track, adding the track on its first chunk. */ + fun write(chunk: MicChunk) { + if (chunk.frames <= 0) return + lock.withLock { + if (closed) return + try { + val clock = clockFor(chunk) ?: return + if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs + val time = clock.stamp(chunk) - originMs + val meta = metadata(chunk) + if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta) + val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels) + writer.writeAudio(clock.number, time, bytes, 0, bytes.size) + lastTimeMs = maxOf(lastTimeMs, time) + } catch (error: Throwable) { + fail(error) + } + } + } + + /** + * Records a JSON object as an event at the recording's current position (a capture starting + * over with another configuration, say), and has every stream's clock placed afresh by its + * next chunk: what follows an event is not continuous with what came before. + */ + fun event(json: String) { + lock.withLock { + if (closed) return + try { + for (clock in clocks.values) clock.reset() + if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json) + } catch (error: Throwable) { + fail(error) + } + } + } + + /** Completes the file. Returns the recording's length in ms. */ + fun finish(): Long { + lock.withLock { + if (!closed) { + closed = true + try { + writer.finish() + } catch (error: Throwable) { + failed = true + logWarn("finishing the recording failed: ${error.message}") + } + } + return writer.durationMs + } + } + + private fun fail(error: Throwable) { + failed = true + closed = true + logWarn("recording stopped: ${error.message}") + try { + writer.finish() + } catch (ignored: Throwable) { + // The file is whatever made it to disk. + } + } + + private fun ensureMetaTrack(): Boolean { + if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK) + return metaTrack > 0 + } + + private fun clockFor(chunk: MicChunk): TrackClock? { + val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong() + clocks[key]?.let { return it } + // The frames track goes in with the first audio track, so it is there for that track's first chunk. + ensureMetaTrack() + // The mixed streams are what a player should pick; a temple's raw pair is for analysis. + val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX + val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred) + if (number <= 0) { + logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio") + return null + } + return TrackClock(number, chunk.sampleRate).also { clocks[key] = it } + } + + /** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */ + private fun metadata(chunk: MicChunk): String? { + if (chunk.stream == MicStream.MIX) return null + if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null + val text = StringBuilder(96) + text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs) + if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence) + if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs) + if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr) + if (chunk.truncated) text.append(",\"trunc\":true") + if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore) + return text.append('}').toString() + } +} + +/** + * Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO] + * chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the + * metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A + * Matroska file from elsewhere plays its first PCM16 track as the mono microphone. + * + * Throws IllegalArgumentException when the file has no audio a replay can use. + */ +class MicRecordingReader(input: SeekableInput) { + companion object { + const val SAMPLE_RATE = 16000 + /** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */ + const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20 + } + + private class Meta(val timeMs: Long, val fields: Map) + + private val reader = MkaReader(input) + private val streams = HashMap() + private var metaTrack = -1 + private val pendingMeta = arrayOfNulls(MicStream.COUNT) + private val queued = ArrayDeque() + + val durationMs: Long get() = reader.durationMs + + /** The capture's "capture.json" attachment, or null. */ + val configJson: String? = + reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString() + + /** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */ + val description: String + + init { + try { + val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 } + for (track in audio) { + val stream = MicStream.named(track.name) + if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream + } + if (streams.isEmpty()) { + // Not one of ours: its first usable track stands in for the microphone. + val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull() + require(first != null) { "no 16-bit PCM audio track" } + streams[first.number] = MicStream.MONO + } + for (number in streams.keys) { + val rate = reader.track(number)!!.sampleRate + require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" } + } + metaTrack = reader.tracks.firstOrNull { + it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK + }?.number ?: -1 + description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) -> + val channels = reader.track(number)!!.channels + MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else "" + } + } catch (error: Throwable) { + reader.close() + throw error + } + } + + /** Continue from the first chunk at or after [timeMs]. */ + fun seekTo(timeMs: Long) { + queued.clear() + pendingMeta.fill(null) + reader.seekTo(timeMs) + } + + /** The next chunk in file order, or null at the end. */ + fun next(): MicChunk? { + while (queued.isEmpty()) { + val block = reader.next() ?: return null + if (block.track == metaTrack) { + noteMeta(block) + continue + } + val stream = streams[block.track] ?: continue + val track = reader.track(block.track) ?: continue + queueChunks(block, stream, track) + } + return queued.removeFirst() + } + + fun close() = reader.close() + + private fun noteMeta(block: MkaBlock) { + try { + val fields = Json.parseObject(block.data.decodeToString()) + val stream = MicStream.named(fields["s"].asString() ?: return) + if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields) + } catch (ignored: Throwable) { + // An event or a frame we can't read: the audio plays without it. + } + } + + private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) { + val sourceChannels = track.channels + val totalFrames = block.data.size / (2 * sourceChannels) + if (totalFrames <= 0) return + // The mixed microphone is one channel however a foreign file stored it. + val channels = if (stream == MicStream.MONO) 1 else sourceChannels + val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs } + pendingMeta[stream] = null + var start = 0 + while (start < totalFrames) { + val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start) + val pcm = ShortArray(frames * channels) + for (frame in 0 until frames) { + val at = (start + frame) * sourceChannels * 2 + if (channels == sourceChannels) { + for (channel in 0 until channels) { + pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1]) + } + } else { + var sum = 0 + for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1]) + pcm[frame] = (sum / sourceChannels).toShort() + } + } + val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate) + if (meta != null && start == 0) applyMeta(chunk, meta.fields) + queued.addLast(chunk) + start += frames + } + } + + private fun applyMeta(chunk: MicChunk, fields: Map) { + chunk.sequence = fields["seq"].asInt() ?: -1 + chunk.deviceTickMs = fields["tick"].asLong() ?: -1L + val angle = fields["angle"].asInt() + if (angle != null) { + chunk.hasDirection = true + chunk.angleDegrees = angle + chunk.ssr = fields["ssr"].asInt() ?: 0 + } + chunk.truncated = fields["trunc"].asBoolean() ?: false + chunk.lostFramesBefore = fields["lost"].asInt() ?: 0 + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt index 8ad954f0..57b6ee8e 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt @@ -35,6 +35,12 @@ interface VoiceCaptureHost { /** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */ fun speakerEmbedder(modelPath: String): SpeakerEmbedder + + /** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */ + fun micArrayLink(): MicArrayLink? = null + + /** Opens a recording to replay; throws when it cannot be read. */ + fun openRecording(path: String): SeekableInput = openFileInput(path) } /** @@ -44,6 +50,12 @@ interface VoiceCaptureHost { * REPLACE-semantics partials, and speaker verification at session end. The session owns its * worker thread (started through the shared [startThread]); listener callbacks go through the * host's [CallbackDispatcher] so the adapter decides the delivery thread. + * + * Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the + * phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording + * played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go + * to the recorder ([setRecorder]) as they came, before any processing, which is what lets a + * replay of the recording run the same processing again. */ class VoiceCaptureSession( private val host: VoiceCaptureHost, @@ -95,6 +107,19 @@ class VoiceCaptureSession( const val DEFAULT_VERIFY_THRESHOLD = 0.8f /** Samples per G2 packet; the decode buffer the worker reuses. */ const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET + // Mic array upkeep: the temples stop streaming unless their lease is renewed, and a + // temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again. + const val ARRAY_CHECK_MS = 1000L + const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3 + const val ARRAY_TEMPLE_SILENT_MS = 5000L + const val ARRAY_WAITING_MS = 2000L + const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..." + // A replay skips stretches of the recording with no audio longer than this + // (a recording made across a disconnect) instead of sitting through them. + const val REPLAY_MAX_GAP_MS = 2000L + const val REPLAY_POSITION_INTERVAL_MS = 1000L + const val REPLAY_WAIT_SLICE_MS = 20L + const val REPLAY_FINISHED_STATUS = "Replay finished." /** Joins transcript pieces, attaching leading punctuation to the previous text. */ @JvmStatic @@ -120,6 +145,8 @@ class VoiceCaptureSession( CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side } + private enum class Source { G2, PHONE, ARRAY, REPLAY } + private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) { val arm: String = arm ?: "?" } @@ -137,11 +164,33 @@ class VoiceCaptureSession( @Volatile private var workerThreadName: String? = null // Capture from the phone's own microphone instead of the G2 over BLE - // (preview-only mode). Latched into activePhoneMic at start() (under `lock`) + // (preview-only mode). Latched into activeSource at start() (under `lock`) // so a mid-session setter call can't switch pipelines underneath the worker. @Volatile private var usePhoneMic = false - private var activePhoneMic = false + // What the next capture reads (see start()); latched into activeSource and its + // companions under `lock`, like usePhoneMic. + @Volatile + private var micArrayConfigure: ByteArray? = null + @Volatile + private var replayPath: String? = null + @Volatile + private var replayStartMs = 0L + private var activeSource = Source.G2 + private var activeConfigure: ByteArray? = null + private var activeReplayPath: String? = null + private var activeReplayStartMs = 0L + // Replay controls, picked up by the worker between chunks. + @Volatile + private var replaySeekMs = -1L + @Volatile + private var replayPaused = false + @Volatile + private var recorder: MicRecordingWriter? = null + @Volatile + private var mixer: MicArrayMixer = SingleTempleMixer() + private var lastMissingPackets = 0L + private var arrayRateWarned = false private var mode = Mode.CLOUD @Volatile private var onboardModelKind = VoiceModelKind.MOONSHINE @@ -193,6 +242,47 @@ class VoiceCaptureSession( this.usePhoneMic = usePhoneMic } + /** + * Capture the next glasses session from the temples' own microphones instead of the stock + * mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples + * (it decides how many microphones each one streams). Null goes back to the mixed stream. + */ + fun setMicArray(configureRecord: ByteArray?) { + this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null + } + + /** How the temples' chunks become the mono feed that is transcribed. */ + fun setMicArrayMixer(mixer: MicArrayMixer) { + this.mixer = mixer + } + + /** + * Have the next capture play the recording at [path] from [startMs] in real time, in place + * of any microphone; null goes back to the microphones. A replay needs no glasses. + */ + fun setReplay(path: String?, startMs: Long) { + this.replayPath = path + this.replayStartMs = maxOf(0L, startMs) + } + + /** Continue the running replay from [positionMs] into its recording. */ + fun seekReplay(positionMs: Long) { + this.replaySeekMs = maxOf(0L, positionMs) + } + + /** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */ + fun setReplayPaused(paused: Boolean) { + this.replayPaused = paused + } + + /** + * Where live captures' chunks are recorded as they arrive, or null. Takes effect at once, + * also mid-capture; the caller finishes the recording. A replay is never recorded. + */ + fun setRecorder(recorder: MicRecordingWriter?) { + this.recorder = recorder + } + fun setOnboardModelKind(kind: String?) { this.onboardModelKind = parseModelKind(kind) } @@ -258,13 +348,26 @@ class VoiceCaptureSession( emitStopped(captureId) return } - if (!usePhoneMic && !host.isG2SessionReady()) { + val path = replayPath + val configure = micArrayConfigure + val source = when { + path != null -> Source.REPLAY + usePhoneMic -> Source.PHONE + configure != null -> Source.ARRAY + else -> Source.G2 + } + if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) { emitStatus("Voice control needs an active G2 connection.") emitStopped(captureId) return } mode = parseMode(requestedMode) - activePhoneMic = usePhoneMic + activeSource = source + activeConfigure = configure + activeReplayPath = path + activeReplayStartMs = replayStartMs + replaySeekMs = -1L + replayPaused = false started = true audioStarted = false val finished = Latch(1, platform) @@ -281,21 +384,22 @@ class VoiceCaptureSession( */ fun isCapturing(): Boolean { var audioUp = false - var phoneMic = false + var source = Source.G2 lock.withLock { if (!started) { return false } audioUp = audioStarted - phoneMic = activePhoneMic + source = activeSource } if (!audioUp) { // The worker is still bringing the mic up; report it as running so // a concurrent request shares it instead of restarting it. return true } - if (phoneMic) { - // The phone mic has no session to lose the enable to; it runs until stop(). + if (source == Source.PHONE || source == Source.REPLAY) { + // Neither has a glasses session to lose the enable to: the phone mic runs + // until stop(), a replay until its recording ends. return true } return host.isG2AudioCaptureActive() @@ -317,7 +421,7 @@ class VoiceCaptureSession( finished = workerFinished workerName = workerThreadName } - stopG2Audio() + stopSource() audioQueue.wake() if (finished != null && (workerName == null || currentThreadName() != workerName)) { finished.await(1500) @@ -356,28 +460,62 @@ class VoiceCaptureSession( val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null verifyCount = 0 - if (activePhoneMic) { - val record = host.openPhoneMic() - if (record == null) { - emitStatus("Could not start the phone microphone.") - return + mixer.reset() + lastMissingPackets = 0 + val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..." + when (activeSource) { + Source.PHONE -> { + val record = host.openPhoneMic() + if (record == null) { + emitStatus("Could not start the phone microphone.") + return + } + lock.withLock { audioStarted = true } + emitStatus(listening) + try { + processPhoneAudio(record) + } finally { + record.close() + } } - lock.withLock { audioStarted = true } - emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...") - try { - processPhoneAudio(record) - } finally { - record.close() + Source.REPLAY -> { + val reader = try { + MicRecordingReader(host.openRecording(activeReplayPath!!)) + } catch (error: Throwable) { + emitStatus("Could not open the recording: " + error.message) + emitReplay("error", 0, 0) + return + } + try { + lock.withLock { audioStarted = true } + logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s") + emitStatus(listening) + processReplay(reader) + } finally { + reader.close() + } } - } else { - packetDecoder = host.createPacketDecoder() - if (!startG2Audio()) { - emitStatus("Could not start G2 microphone input.") - return + Source.ARRAY -> { + val link = host.micArrayLink() + val configure = activeConfigure + if (link == null || configure == null || !startArray(link, configure)) { + emitStatus("Could not start the glasses microphone array.") + return + } + lock.withLock { audioStarted = true } + emitStatus(listening) + processArray(link, configure, listening) + } + Source.G2 -> { + packetDecoder = host.createPacketDecoder() + if (!startG2Audio()) { + emitStatus("Could not start G2 microphone input.") + return + } + lock.withLock { audioStarted = true } + emitStatus(listening) + processG2Audio() } - lock.withLock { audioStarted = true } - emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...") - processG2Audio() } // Verification result must precede the final transcript so the // TS bridge can suppress a non-wearer command before it is acted @@ -392,7 +530,7 @@ class VoiceCaptureSession( logError("Voice control failed", error) emitStatus("Voice control failed: " + error.message) } finally { - stopG2Audio() + stopSource() releaseRecognizer() releaseDecoder() lock.withLock { @@ -432,23 +570,214 @@ class VoiceCaptureSession( continue } decodedSamples += count.toLong() - // The beam filter drops packets whose firmware direction-of-arrival - // falls outside the listening wedge (isolating the aimed talker for - // every consumer, recognition included); the spectral noise - // suppressor then cleans what remains before it reaches the - // recognizer, cloud PCM, endpointing, or speaker verification. - val angleDegrees = currentDecoder.lastAngleDegrees - val ssr = currentDecoder.lastSsr - if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { - emitFrameMeta(angleDegrees, ssr) - maybeEmitAudioStats(false) - continue - } - processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta) + val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs) + chunk.hasDirection = currentDecoder.hasFrameMeta + chunk.angleDegrees = currentDecoder.lastAngleDegrees + chunk.ssr = currentDecoder.lastSsr + // Packets the decoder found missing ahead of this one: a gap in a recording. + val missing = currentDecoder.missingPackets + chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt() + lastMissingPackets = missing + handleMonoChunk(chunk) maybeEmitAudioStats(false) } } + /** + * A mixed-microphone chunk from any source: recorded as it came, then on to processing + * unless the beam filter drops it. The filter drops chunks whose firmware + * direction-of-arrival falls outside the listening wedge (isolating the aimed talker for + * every consumer, recognition included); the spectral noise suppressor then cleans what + * remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification. + */ + private fun handleMonoChunk(chunk: MicChunk) { + recordChunk(chunk) + if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) { + emitFrameMeta(chunk.angleDegrees, chunk.ssr) + return + } + processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection) + } + + /** + * A chunk of one temple's own microphones: recorded as it came, then mixed into the mono + * feed, which is recorded too (as what was transcribed) and processed like a mixed chunk. + * The beam filter does not apply: a temple's bearing is along its own pair, not the + * head-frame direction the filter is aimed in. + */ + private fun handleArrayChunk(chunk: MicChunk) { + recordChunk(chunk) + val feed = mixer.mix(chunk) ?: return + recordChunk(feed) + if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) { + if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz") + arrayRateWarned = true + return + } + processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection) + } + + private fun recordChunk(chunk: MicChunk) { + if (activeSource != Source.REPLAY) recorder?.write(chunk) + } + + private fun handleChunk(chunk: MicChunk) { + if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk) + } + + private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean { + resetAudioStats() + audioQueue.clear() + if (!startArrayForwarding(link)) return false + return link.sendControl(configure, "configure", true, true) + } + + private fun startArrayForwarding(link: MicArrayLink): Boolean = + link.startForwarding(object : FaceclawAudioPacketListener { + override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + queueAudioPacket(data, arm, arrivalMs) + } + }) + + /** + * Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream. + * Between frames it keeps the streams alive: renews the temples' lease, configures a temple + * again when it has gone quiet, and takes forwarding back when the glasses session dropped it. + */ + private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) { + val startedAt = platform.elapsedRealtimeMs() + var checkedAt = startedAt + var renewedAt = startedAt + // Left, right. + val armedAt = longArrayOf(startedAt, startedAt) + val frameAt = longArrayOf(startedAt, startedAt) + var waiting = false + while (started) { + val packet = takeAudioPacket() + val now = platform.elapsedRealtimeMs() + if (packet != null) { + val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1 + val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null + val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null + // Anything else is a stock-format packet from before arming took effect. + if (frame != null && pcm != null) { + frameAt[side] = now + val frames = pcm.size / frame.channels + decodedSamples += frames.toLong() + val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT + val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs) + chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0 + chunk.angleDegrees = frame.angleDegrees + chunk.ssr = frame.ssr + chunk.sequence = frame.sequence + chunk.deviceTickMs = frame.deviceTickMs + chunk.truncated = frame.truncated + handleArrayChunk(chunk) + } + } + if (now - checkedAt >= ARRAY_CHECK_MS) { + checkedAt = now + if (!link.isForwarding()) { + if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) { + logInfo("mic array forwarding re-established") + armedAt.fill(now) + renewedAt = now + } + } else { + if (now - renewedAt >= ARRAY_RENEW_MS) { + link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true) + renewedAt = now + } + for (side in 0..1) { + if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue + logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again") + link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0) + armedAt[side] = now + } + } + val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS + if (silent != waiting) { + waiting = silent + emitStatus(if (silent) ARRAY_WAITING_STATUS else listening) + } + } + maybeEmitAudioStats(false) + } + } + + /** + * Replay loop: hands the recording's chunks to the pipeline at the pace they were captured, + * so recognition (its partials, a cloud provider's stream) sees what a live capture would. + * Jumps and pauses asked for from outside take effect between chunks. + */ + private fun processReplay(reader: MicRecordingReader) { + val duration = reader.durationMs + var position = minOf(activeReplayStartMs, duration) + reader.seekTo(position) + emitReplay("start", position, duration) + var reportedAt = position + // Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap. + var anchorWall = -1L + var anchorFile = 0L + var pending: MicChunk? = null + while (started) { + val seek = replaySeekMs + if (seek >= 0) { + replaySeekMs = -1L + // What played so far is a finished utterance; the rest starts clean. + finishReplayedUtterance() + position = minOf(seek, duration) + reader.seekTo(position) + pending = null + anchorWall = -1L + reportedAt = position + emitReplay("seek", position, duration) + } + if (replayPaused) { + anchorWall = -1L + sleepMs(REPLAY_WAIT_SLICE_MS) + continue + } + val chunk = pending ?: reader.next() + if (chunk == null) { + emitReplay("end", duration, duration) + emitStatus(REPLAY_FINISHED_STATUS) + return + } + pending = chunk + val now = platform.elapsedRealtimeMs() + if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) { + anchorWall = now + anchorFile = chunk.timeMs + } + val wait = anchorWall + (chunk.timeMs - anchorFile) - now + if (wait > 0) { + // Short slices, so a stop, jump or pause is noticed promptly. + sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS)) + continue + } + pending = null + position = chunk.timeMs + decodedSamples += chunk.frames.toLong() + handleChunk(chunk) + if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) { + reportedAt = position + emitReplay("position", position, duration) + } + } + } + + /** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */ + private fun finishReplayedUtterance() { + mixer.reset() + endpointDetector.reset() + suppressor?.reset() + if (mode != Mode.ONBOARD) return + decodeTranscript(true) + resetTranscriptState() + lastTranscript = "" + } + /** * Per-chunk processing shared by the G2 and phone-mic paths, downstream of * decode and the beam filter. hasFrameMeta is false for the phone mic, @@ -495,7 +824,7 @@ class VoiceCaptureSession( continue } decodedSamples += read.toLong() - processPcmChunk(pcm, read, 0, 0, false) + handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs())) } } @@ -681,6 +1010,11 @@ class VoiceCaptureSession( host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) } } + private fun emitReplay(event: String, positionMs: Long, durationMs: Long) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) } + } + private fun emitSpeechEnd() { val currentListener = listener ?: return host.dispatcher.post { currentListener.onSpeechEnd() } @@ -691,8 +1025,21 @@ class VoiceCaptureSession( host.dispatcher.post { currentListener.onStopped(captureId) } } - private fun stopG2Audio() { - host.g2AudioSource()?.stop() + /** + * Release what the running capture's source holds on the glasses. Runs on stop() and again + * as the worker winds down (it may have started the source after stop() looked), so + * everything here is safe to repeat. + */ + private fun stopSource() { + val source = lock.withLock { activeSource } + when (source) { + Source.G2 -> host.g2AudioSource()?.stop() + Source.ARRAY -> host.micArrayLink()?.let { link -> + link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true) + link.stopForwarding() + } + Source.PHONE, Source.REPLAY -> {} + } maybeEmitAudioStats(true) } @@ -711,7 +1058,7 @@ class VoiceCaptureSession( if (!started || data == null) { return } - if ("L" != arm) { + if ("L" != arm && activeSource != Source.ARRAY) { wrongArmPackets++ } audioQueue.put(AudioPacket(data, arm, arrivalMs)) { diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt index 2c0bdd9a..fe2dba95 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt @@ -67,6 +67,24 @@ interface PacketAudioSource { fun isActive(): Boolean } +/** + * The custom firmware's mic_control channel on a glasses session: each temple streams its own + * microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed + * stream uses, while it holds a configuration that the phone keeps renewing. + */ +interface MicArrayLink { + /** Deliver the audio characteristic's packets to [listener]; false when no session is up. */ + fun startForwarding(listener: FaceclawAudioPacketListener): Boolean + + fun stopForwarding() + + /** Whether forwarding still holds: it lives in the glasses session, which can drop it. */ + fun isForwarding(): Boolean + + /** Queue a control record for the chosen temples; false when nothing was queued. */ + fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean +} + /** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */ interface PcmAudioSource { /** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */ diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt index e3b4fd75..80736519 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawVoiceControllerListener.kt @@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener { * speaker model could not be loaded. */ fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit + + /** + * Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay): + * "start" once the file is open, "position" about once per second of it played, "seek" after + * a jump took effect (the transcript of what played before it has been emitted as final), + * "end" when the file ran out, and "error" when it could not be opened (the status says + * why). Positions are milliseconds into the file. + */ + fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit } diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosMicRecorder.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosMicRecorder.kt new file mode 100644 index 00000000..4dd5aa05 --- /dev/null +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosMicRecorder.kt @@ -0,0 +1,37 @@ +package com.faceclaw.app + +import platform.Foundation.NSData + +/** + * Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's + * captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record: + * 16 kHz mono PCM16, as its recognizer hands it to TypeScript. + */ +class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) { + /** The recording stopped early: the file could not be written. */ + val failed: Boolean get() = writer.failed + + /** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */ + fun append(pcm16le: NSData) { + val bytes = pcm16le.byteArray() + val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) } + writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs())) + } + + /** Completes the file. Returns the recording's length in ms. */ + fun finish(): Long = writer.finish() +} + +object IosMicRecordings { + /** + * A recorder writing a new file at [path], or null when the file cannot be created. + * [configJson] describes the capture and is stored in the file. + */ + fun open(path: String, configJson: String?): IosMicRecorder? = + try { + IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform)) + } catch (error: Throwable) { + PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}") + null + } +} diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt index b89c02ff..7dadbbf4 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt @@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped { actual fun deleteFile(path: String): Boolean = unlink(path) == 0 actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0 + +actual fun openFileOutput(path: String): SeekableOutput { + val file = fopen(path, "wb+") ?: error("Cannot write $path") + return object : SeekableOutput { + override var size = 0L + private set + + private fun put(bytes: ByteArray, offset: Int, length: Int) { + if (length == 0) return + bytes.usePinned { + check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert()) { "Short write to $path" } + } + } + + override fun write(bytes: ByteArray, offset: Int, length: Int) { + put(bytes, offset, length) + size += length + // Each write is a complete element; hand it to the OS so a crash keeps it. + platform.posix.fflush(file) + } + + override fun writeAt(position: Long, bytes: ByteArray) { + require(position >= 0 && position + bytes.size <= size) + check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0) + put(bytes, 0, bytes.size) + check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0) + platform.posix.fflush(file) + } + + override fun close() { + fclose(file) + } + } +} + +actual fun openFileInput(path: String): SeekableInput { + val file = fopen(path, "rb") ?: error("Cannot read $path") + val fileSize = fileLength(path) + return object : SeekableInput { + override val length = fileSize + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int { + if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0 + return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() } + } + + override fun close() { + fclose(file) + } + } +} diff --git a/tests/ios-voice.test.cjs b/tests/ios-voice.test.cjs index 2561c249..d0c533de 100644 --- a/tests/ios-voice.test.cjs +++ b/tests/ios-voice.test.cjs @@ -36,7 +36,7 @@ function harness() { static open(name) { recordingAttempts++; if (failRecordings) return null; - const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; + const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.mka`; r.pcm = []; r.closed = false; recordings.push(r); return r; } append(bytes) { this.pcm.push(...bytes); } close() { this.closed = true; } @@ -49,7 +49,7 @@ function harness() { setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn), NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d }, require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer - : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} }; + : name === './mic-recorder.ios' ? { MicRecorder: FakeRecorder } : {} }; vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox); const bridge = new sandbox.exports.IosVoiceControlBridge(); diff --git a/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/MicRecordingFileTest.kt b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/MicRecordingFileTest.kt new file mode 100644 index 00000000..ea454599 --- /dev/null +++ b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/MicRecordingFileTest.kt @@ -0,0 +1,63 @@ +package com.faceclaw.app + +import java.io.File +import kotlin.math.PI +import kotlin.math.sin +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class MicRecordingFileTest { + /** + * A capture recorded through the real file adapters and read back. Set FACECLAW_MKA_SAMPLE + * to a path to keep the file, for checking it against other Matroska tools (ffprobe, mpv). + */ + @Test + fun recordsACaptureToAFileThatReadsBack() { + val kept = System.getenv("FACECLAW_MKA_SAMPLE") + val file = if (kept.isNullOrEmpty()) File.createTempFile("capture", ".mka") else File(kept) + try { + val writer = MicRecordingWriter(openFileOutput(file.path), "{\"mics\":4,\"app\":\"test\"}", 1_700_000_000_000) + val mixer = SingleTempleMixer() + // Five seconds: each temple streams a stereo pair of tones on its own tick. + for (index in 0 until 100) { + for (side in 0..1) { + val pcm = ShortArray(1600) { + val frame = index * 800 + it / 2 + val hz = if (side == 0) 440.0 else 660.0 + (sin(2 * PI * hz * frame / 16000) * (if (it % 2 == 0) 8000 else 4000)).toInt().toShort() + } + val chunk = MicChunk(if (side == 0) MicStream.LEFT else MicStream.RIGHT, 2, 16000, pcm, 800, 10_000 + index * 50L + side * 7) + chunk.sequence = index + chunk.deviceTickMs = (if (side == 0) 500_000L else 123_456L) + index * 50L + chunk.hasDirection = true + chunk.angleDegrees = if (side == 0) -30 else 25 + chunk.ssr = 12 + writer.write(chunk) + mixer.mix(chunk)?.let { writer.write(it) } + } + } + // The right temple's chunks arrive 7 ms behind the left's. + assertEquals(5007L, writer.finish()) + + val reader = MicRecordingReader(openFileInput(file.path)) + assertEquals(5007L, reader.durationMs) + assertEquals("left temple 2ch + right temple 2ch", reader.description) + assertEquals("{\"mics\":4,\"app\":\"test\"}", reader.configJson) + var left = 0 + var right = 0 + while (true) { + val chunk = reader.next() ?: break + if (chunk.stream == MicStream.LEFT) left++ else right++ + assertEquals(12, chunk.ssr) + } + assertEquals(100, left) + assertEquals(100, right) + reader.close() + // Audio (three tracks) plus the frame metadata. + assertTrue(file.length() > 100 * 2 * 3200 + 100 * 1600) + } finally { + if (kept.isNullOrEmpty()) file.delete() + } + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/MicRecordingTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/MicRecordingTest.kt new file mode 100644 index 00000000..10e6c50d --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/MicRecordingTest.kt @@ -0,0 +1,572 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** A file held in memory: written through [SeekableOutput], read back through [input]. */ +internal class MemoryFile : SeekableOutput { + var bytes = ByteArray(0) + var closed = false + override val size: Long get() = bytes.size.toLong() + + override fun write(bytes: ByteArray, offset: Int, length: Int) { + this.bytes += bytes.copyOfRange(offset, offset + length) + } + + override fun writeAt(position: Long, bytes: ByteArray) { + bytes.copyInto(this.bytes, position.toInt()) + } + + override fun close() { + closed = true + } + + /** The file as it stands, or its first [length] bytes. */ + fun input(length: Int = bytes.size): SeekableInput { + val snapshot = bytes.copyOf(length) + return object : SeekableInput { + override val length = snapshot.size.toLong() + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int { + val count = minOf(length.toLong(), maxOf(0L, snapshot.size - position)).toInt() + snapshot.copyInto(buffer, offset, position.toInt(), position.toInt() + count) + return count + } + + override fun close() {} + } + } +} + +private fun pcmBytes(frames: Int, channels: Int, value: Int): ByteArray = + AudioSegmentation.pcm16ToLittleEndian(ShortArray(frames * channels) { value.toShort() }, 0, frames * channels) + +private fun chunk(stream: Int, timeMs: Long, value: Int, channels: Int = 1, frames: Int = 800): MicChunk = + MicChunk(stream, channels, 16000, ShortArray(frames * channels) { (value + it % channels).toShort() }, frames, timeMs) + +private fun drain(reader: MicRecordingReader): List { + val out = ArrayList() + while (true) out.add(reader.next() ?: break) + return out +} + +class MatroskaTest { + @Test + fun roundTripsTracksBlocksAndAttachments() { + val file = MemoryFile() + val writer = MkaWriter(file, "session", 1_700_000_000_000, listOf(MkaAttachment("capture.json", "application/json", "{\"a\":1}".encodeToByteArray()))) + val temple = writer.addAudioTrack("left temple", 16000, 2, false) + val text = writer.addTextTrack("frames") + for (index in 0 until 30) { + writer.writeText(text, index * 50L, 50, "{\"n\":$index}") + writer.writeAudio(temple, index * 50L, pcmBytes(800, 2, index), 0, 3200) + } + // A track that first appears mid-file, and is the one players should pick. + val mix = writer.addAudioTrack("mix", 16000, 1, true) + for (index in 30 until 60) { + writer.writeAudio(temple, index * 50L, pcmBytes(800, 2, index), 0, 3200) + writer.writeAudio(mix, index * 50L, pcmBytes(800, 1, -index), 0, 1600) + } + assertEquals(3000L, writer.durationMs) + writer.finish() + assertTrue(file.closed) + + val reader = MkaReader(file.input()) + // Listed with the preferred audio track first, then the others, then text; numbers are as added. + assertEquals(listOf("mix", "left temple", "frames"), reader.tracks.map { it.name }) + assertEquals(listOf(mix, temple, text), reader.tracks.map { it.number }) + val templeInfo = reader.track(temple)!! + assertTrue(templeInfo.isPcm16) + assertEquals(2, templeInfo.channels) + assertEquals(16000, templeInfo.sampleRate) + assertFalse(reader.track(text)!!.isPcm16) + assertEquals("{\"a\":1}", reader.attachments.single().data.decodeToString()) + assertEquals("application/json", reader.attachments.single().mimeType) + assertEquals(3000L, reader.durationMs) + + val blocks = ArrayList() + while (true) blocks.add(reader.next() ?: break) + assertEquals(30 + 60 + 30, blocks.size) + assertEquals(text, blocks[0].track) + assertEquals("{\"n\":0}", blocks[0].data.decodeToString()) + assertEquals(50L, blocks[0].durationMs) + val templeBlocks = blocks.filter { it.track == temple } + assertEquals((0 until 60).map { it * 50L }, templeBlocks.map { it.timeMs }) + assertEquals(pcmBytes(800, 2, 59).toList(), templeBlocks.last().data.toList()) + assertEquals(1500L, blocks.first { it.track == mix }.timeMs) + + reader.seekTo(2000) + val afterSeek = ArrayList() + while (true) afterSeek.add(reader.next() ?: break) + assertEquals(2000L, afterSeek.first().timeMs) + assertEquals(40, afterSeek.size) + reader.seekTo(99_000) + assertNull(reader.next()) + } + + @Test + fun recordingCutShortIsReadableUpToItsLastCompleteBlock() { + val file = MemoryFile() + val writer = MkaWriter(file, null, 0, emptyList()) + val track = writer.addAudioTrack("mic", 16000, 1, true) + // 2.5 s: clusters of a second each reach the file as they fill; the last half-second is still buffered. + for (index in 0 until 50) writer.writeAudio(track, index * 50L, pcmBytes(800, 1, index), 0, 1600) + val whole = MkaReader(file.input()) + assertEquals("mic", whole.tracks.single().name) + var count = 0 + while (whole.next() != null) count++ + assertEquals(40, count) + assertEquals(2000L, whole.durationMs) + // A crash mid-write tears the last cluster: its complete blocks still read. + val torn = MkaReader(file.input(file.bytes.size - 1000)) + count = 0 + while (torn.next() != null) count++ + assertEquals(39, count) + assertEquals(1950L, torn.durationMs) + } + + @Test + fun rejectsOtherFiles() { + val wav = MemoryFile() + wav.write(BinaryEncoding.wavHeader(0, 16000, 1), 0, 44) + assertFailsWith { MkaReader(wav.input()) } + assertFailsWith { MkaReader(MemoryFile().input()) } + } + + @Test + fun paddedRegionsFillTheirReservedSpaceWithAVoidElement() { + assertNull(EbmlWriter.padded(ByteArray(10), 9)) + assertNull(EbmlWriter.padded(ByteArray(10), 11)) + assertEquals(listOf(1, 2, 0xEC, 0x80), EbmlWriter.padded(byteArrayOf(1, 2), 4)!!.map { it.toInt() and 0xff }) + val long = EbmlWriter.padded(ByteArray(0), 300)!! + assertEquals(300, long.size) + // An eight-byte size field: 300 - 9 = 0x123. + assertEquals(listOf(0xEC, 0x01, 0, 0, 0, 0, 0, 0x01, 0x23), long.take(9).map { it.toInt() and 0xff }) + } +} + +class MicRecordingTest { + private val warnings = mutableListOf() + + private fun recorder(file: MemoryFile, config: String? = "{\"mics\":4}") = + MicRecordingWriter(file, config, 1_700_000_000_000, testPlatform()) { warnings.add(it) } + + @Test + fun recordsEveryStreamWithItsMetadataAndReplaysTheSources() { + val file = MemoryFile() + val writer = recorder(file) + for (index in 0 until 4) { + val left = chunk(MicStream.LEFT, 5000 + index * 50L, 100 + index, channels = 2) + left.sequence = 7 + index + left.deviceTickMs = 900_000 + index * 50L + left.hasDirection = true + left.angleDegrees = -12 + left.ssr = 40 + left.truncated = index == 3 + writer.write(left) + val feed = SingleTempleMixer().mix(left)!! + writer.write(feed) + val right = chunk(MicStream.RIGHT, 5010 + index * 50L, 200 + index, channels = 2) + right.sequence = 3 + index + right.deviceTickMs = 40_000 + index * 50L + writer.write(right) + } + // The capture went back to the mixed stream part-way through. + writer.event("{\"event\":\"capture\",\"mics\":1}") + val mono = chunk(MicStream.MONO, 5200, 55) + mono.hasDirection = true + mono.angleDegrees = 30 + mono.ssr = 9 + writer.write(mono) + assertEquals(250L, writer.finish()) + assertFalse(writer.failed) + assertTrue(warnings.isEmpty(), warnings.toString()) + + val mka = MkaReader(file.input()) + // What a player picks by default is a mixed track. + assertEquals(listOf("mix", "mic", "left temple", "right temple", "frames"), mka.tracks.map { it.name }) + + val reader = MicRecordingReader(file.input()) + assertEquals("{\"mics\":4}", reader.configJson) + assertEquals("mic + left temple 2ch + right temple 2ch", reader.description) + assertEquals(250L, reader.durationMs) + val chunks = drain(reader) + // The mix is derived again on a replay, so only the sources come back. + assertEquals(4, chunks.count { it.stream == MicStream.LEFT }) + assertEquals(4, chunks.count { it.stream == MicStream.RIGHT }) + assertEquals(1, chunks.count { it.stream == MicStream.MONO }) + val left = chunks.filter { it.stream == MicStream.LEFT } + assertEquals(listOf(0L, 50L, 100L, 150L), left.map { it.timeMs }) + assertEquals(listOf(7, 8, 9, 10), left.map { it.sequence }) + assertEquals(900_050L, left[1].deviceTickMs) + assertEquals(2, left[0].channels) + assertEquals(listOf(100, 101, 100, 101), left[0].pcm.take(4)) + assertTrue(left.all { it.hasDirection && it.angleDegrees == -12 && it.ssr == 40 }) + assertEquals(listOf(false, false, false, true), left.map { it.truncated }) + val right = chunks.filter { it.stream == MicStream.RIGHT } + assertEquals(listOf(10L, 60L, 110L, 160L), right.map { it.timeMs }) + assertFalse(right[0].hasDirection) + val replayedMono = chunks.single { it.stream == MicStream.MONO } + assertEquals(200L, replayedMono.timeMs) + assertEquals(30, replayedMono.angleDegrees) + assertEquals(800, replayedMono.frames) + + reader.seekTo(100) + assertEquals(100L, reader.next()!!.timeMs) + } + + @Test + fun chunksHeldUpInDeliveryLandWhereTheyWereCaptured() { + val file = MemoryFile() + val writer = recorder(file, null) + // A temple's ticks say when each chunk was captured. The first ten arrive as captured; + // the link then stalls for a second and delivers the next twenty at once. + for (index in 0 until 30) { + val arrival = if (index < 10) 1000 + index * 50L else maxOf(2500L, 1000 + index * 50L) + val chunk = chunk(MicStream.LEFT, arrival, index) + chunk.deviceTickMs = 77_000 + index * 50L + writer.write(chunk) + } + // Two chunks were lost on the way (their ticks are skipped): a gap, not a splice. + val late = chunk(MicStream.LEFT, 2650, 99) + late.deviceTickMs = 77_000 + 32 * 50L + writer.write(late) + writer.finish() + val times = drain(MicRecordingReader(file.input())).map { it.timeMs } + assertEquals((0 until 30).map { it * 50L } + 1600L, times) + assertNull(MicRecordingReader(file.input()).configJson) + } + + @Test + fun aSourceWithoutAClockIsPlacedBySampleCount() { + val file = MemoryFile() + val writer = recorder(file) + for (index in 0 until 10) { + // Arrivals jitter around the capture cadence; the stream's samples are contiguous. + val jitter = if (index % 3 == 1) 30L else 0L + writer.write(chunk(MicStream.MONO, 400 + index * 50L + jitter, index)) + } + // The decoder reported one packet missing before this chunk. + val afterLoss = chunk(MicStream.MONO, 400 + 11 * 50L, 11) + afterLoss.lostFramesBefore = 800 + writer.write(afterLoss) + // A capture that resumes much later is placed by its arrival, not spliced on. + writer.write(chunk(MicStream.MONO, 60_000, 12)) + writer.finish() + val chunks = drain(MicRecordingReader(file.input())) + assertEquals((0 until 10).map { it * 50L } + 550L + 59_600L, chunks.map { it.timeMs }) + assertEquals(800, chunks[10].lostFramesBefore) + } + + @Test + fun aForeignFilePlaysItsFirstTrackAsTheMicrophone() { + val file = MemoryFile() + val mka = MkaWriter(file, null, 0, emptyList()) + val track = mka.addAudioTrack("Stereo", 16000, 2, false) + // One long block of stereo frames [100, 300]. + val samples = ShortArray(2000 * 2) { if (it % 2 == 0) 100 else 300 } + mka.writeAudio(track, 0, AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size), 0, samples.size * 2) + mka.finish() + val reader = MicRecordingReader(file.input()) + assertEquals("mic 2ch", reader.description) + val chunks = drain(reader) + assertEquals(listOf(800, 800, 400), chunks.map { it.frames }) + assertEquals(listOf(0L, 50L, 100L), chunks.map { it.timeMs }) + assertTrue(chunks.all { it.stream == MicStream.MONO && it.channels == 1 && it.pcm.all { sample -> sample == 200.toShort() } }) + + val fast = MemoryFile() + val other = MkaWriter(fast, null, 0, emptyList()) + other.writeAudio(other.addAudioTrack("mic", 48000, 1, true), 0, pcmBytes(480, 1, 1), 0, 960) + other.finish() + val error = assertFailsWith { MicRecordingReader(fast.input()) } + assertTrue(error.message!!.contains("48000 Hz"), error.message) + } + + @Test + fun theFeedFollowsTheLeftTempleAndFallsBackToTheRight() { + val mixer = SingleTempleMixer() + // Until the left temple is heard from, the right one feeds. + assertNotNull(mixer.mix(chunk(MicStream.RIGHT, 0, 10, channels = 2))) + val left = chunk(MicStream.LEFT, 20, 100, channels = 2) + left.deviceTickMs = 5 + val feed = mixer.mix(left)!! + assertEquals(MicStream.MIX, feed.stream) + assertEquals(1, feed.channels) + assertEquals(800, feed.frames) + // Channels [100, 101] average to 100. + assertTrue(feed.pcm.all { it == 100.toShort() }) + assertEquals(MicStream.LEFT, feed.clockStream) + assertEquals(5L, feed.deviceTickMs) + assertNull(mixer.mix(chunk(MicStream.RIGHT, 60, 10))) + assertNull(mixer.mix(chunk(MicStream.RIGHT, 300, 10))) + // The left temple went quiet: the right stands in until it is back. + assertNotNull(mixer.mix(chunk(MicStream.RIGHT, 320, 10))) + assertNotNull(mixer.mix(chunk(MicStream.LEFT, 330, 100))) + assertNull(mixer.mix(chunk(MicStream.RIGHT, 340, 10))) + assertNull(mixer.mix(chunk(MicStream.MONO, 350, 10))) + } + + @Test + fun decodesTheTemplesStreamFrames() { + val frame = micStreamFrame(sequence = 0x1234, tick = 0x89ABCDEFL, channels = 2, samples = shortArrayOf(1, -2, 3, -4), angle = -45, ssr = 300, truncated = true) + val decoded = MicArrayProtocol.decodeFrame(frame)!! + assertEquals(0x1234, decoded.sequence) + assertEquals(0x89ABCDEFL, decoded.deviceTickMs) + assertEquals(2, decoded.channels) + assertEquals(16000, decoded.sampleRate) + assertEquals(-45, decoded.angleDegrees) + assertEquals(300, decoded.ssr) + assertTrue(decoded.truncated) + assertEquals(listOf(1, -2, 3, -4), MicArrayProtocol.pcm16(frame, decoded)!!.toList()) + assertNull(MicArrayProtocol.decodeFrame(ByteArray(205))) + // A payload length beyond the packet is cut to what arrived. + val short = frame.copyOf(frame.size - 4) + assertEquals(listOf(1, -2), MicArrayProtocol.pcm16(short, MicArrayProtocol.decodeFrame(short)!!)!!.toList()) + assertEquals(listOf(0x4d, 0x43, 1, 4), MicArrayProtocol.control(MicArrayProtocol.OP_RENEW).map { it.toInt() }) + assertTrue(MicArrayProtocol.isConfigure(CONFIGURE_RECORD)) + assertFalse(MicArrayProtocol.isConfigure(MicArrayProtocol.control(MicArrayProtocol.OP_STOP))) + } +} + +/** A mic_control CONFIGURE record: codec front end, both mics, raw PCM16 at 16 kHz, hardware armed. */ +internal val CONFIGURE_RECORD = byteArrayOf(0x4d, 0x43, 1, 1, 0, 3, 1, 0, 160.toByte(), 0, 0, 0, 2) + +/** An 'SM' stream frame as a temple sends it. */ +internal fun micStreamFrame( + sequence: Int, + tick: Long, + channels: Int, + samples: ShortArray, + angle: Int = 0, + ssr: Int = 0, + truncated: Boolean = false, +): ByteArray { + val payload = AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size) + val header = ByteArray(MicArrayProtocol.HEADER_BYTES) + fun le16(at: Int, value: Int) { + header[at] = value.toByte() + header[at + 1] = (value shr 8).toByte() + } + header[0] = 'S'.code.toByte() + header[1] = 'M'.code.toByte() + header[2] = 1 + header[3] = (if (truncated) 0x82 else 0x02).toByte() + le16(4, sequence) + le16(6, (tick and 0xffff).toInt()) + le16(8, ((tick shr 16) and 0xffff).toInt()) + header[10] = channels.toByte() + le16(11, 160) + header[13] = 0 + header[14] = 1 + le16(15, angle) + le16(17, ssr) + le16(19, payload.size) + return header + payload +} + +class VoiceCaptureSourcesTest { + private class Events : FaceclawVoiceControllerListener { + val statuses = mutableListOf() + val transcripts = mutableListOf>() + val pcmChunks = mutableListOf() + val replay = mutableListOf>() + val stoppedLatch = Latch(1, testPlatform()) + override fun onStatus(status: String?) { statuses.add(status) } + override fun onTranscript(text: String?, isFinal: Boolean) { transcripts.add(text to isFinal) } + override fun onStopped(captureId: Int) { stoppedLatch.countDown() } + override fun onPcm(pcm16le: ByteArray?) { pcmChunks.add(pcm16le!!) } + override fun onFrameMeta(angleDegrees: Int, ssr: Int) {} + override fun onSpeechEnd() {} + override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) {} + override fun onReplay(event: String?, positionMs: Long, durationMs: Long) { replay.add(event to positionMs) } + } + + private class FakeLink : MicArrayLink { + var listener: FaceclawAudioPacketListener? = null + val controls = mutableListOf() + override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean { this.listener = listener; return true } + override fun stopForwarding() { listener = null } + override fun isForwarding(): Boolean = listener != null + override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean { + controls.add(label + ":" + record[3] + (if (leftTemple) "L" else "") + (if (rightTemple) "R" else "")) + return true + } + } + + private class FakeHost(val link: FakeLink? = null, val recording: MemoryFile? = null) : VoiceCaptureHost { + override val dispatcher = CallbackDispatcher { it() } + override fun onSessionStarting() {} + override fun isG2SessionReady(): Boolean = link != null + override fun isG2AudioCaptureActive(): Boolean = link?.isForwarding() == true + override fun g2AudioSource(): PacketAudioSource? = null + override fun createPacketDecoder(): AudioPacketDecoder = error("no stock stream in these tests") + override fun openPhoneMic(): PcmAudioSource? = null + override fun hasTranscriberModel(kind: VoiceModelKind): Boolean = true + override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber = object : OfflineTranscriber { + override fun recognize(samples: FloatArray, count: Int): String = "x" + override fun release() {} + } + override fun speakerEmbedder(modelPath: String): SpeakerEmbedder = error("no verification in these tests") + override fun micArrayLink(): MicArrayLink? = link + override fun openRecording(path: String): SeekableInput = if (path == "missing") error("no such file") else recording!!.input() + } + + private val logs = mutableListOf() + + private fun session(host: VoiceCaptureHost, name: String) = + VoiceCaptureSession(host, testPlatform(), name, { logs.add("I $it") }, { logs.add("W $it") }, { m, e -> logs.add("E $m ${e.message}") }) + + private fun waitFor(timeoutMs: Long, condition: () -> Boolean): Boolean { + val deadline = testPlatform().elapsedRealtimeMs() + timeoutMs + while (!condition()) { + if (testPlatform().elapsedRealtimeMs() > deadline) return false + sleepMs(5) + } + return true + } + + private fun monoRecording(chunks: Int): MemoryFile { + val file = MemoryFile() + val writer = MicRecordingWriter(file, null, 0, testPlatform()) + for (index in 0 until chunks) writer.write(chunk(MicStream.MONO, index * 50L, 1000 + index)) + writer.finish() + return file + } + + @Test + fun micArrayCaptureArmsTheTemplesRecordsThemAndTranscribesTheFeed() { + val link = FakeLink() + val host = FakeHost(link) + val events = Events() + val session = session(host, "voice-test-array") + val file = MemoryFile() + val recorder = MicRecordingWriter(file, "{}", 0, testPlatform()) + session.setListener(events) + session.setMicArray(CONFIGURE_RECORD) + session.setRecorder(recorder) + session.start("cloud", 1) + assertTrue(waitFor(3000) { link.controls.isNotEmpty() }) + assertTrue(session.isCapturing()) + val listener = link.listener!! + for (index in 0 until 5) { + val left = ShortArray(1600) { if (it % 2 == 0) 100 else 300 } + listener.onAudioPacket(micStreamFrame(index, 1000 + index * 50L, 2, left, angle = 20, ssr = 7), "L", 500 + index * 50L) + listener.onAudioPacket(micStreamFrame(index, 9000 + index * 50L, 2, ShortArray(1600) { 50 }), "R", 505 + index * 50L) + } + // A stock-format packet from before arming took effect is not audio. + listener.onAudioPacket(ByteArray(205), "L", 760) + assertTrue(waitFor(3000) { events.pcmChunks.size >= 5 }) + session.stop() + assertTrue(events.stoppedLatch.await(3000)) + assertEquals(listOf("configure:1LR"), link.controls.take(1)) + assertEquals("stop:3LR", link.controls.last()) + assertFalse(link.isForwarding()) + assertFalse(session.isCapturing()) + // The feed is the left temple's microphones averaged. + assertEquals(5, events.pcmChunks.size) + assertEquals(1600, events.pcmChunks[0].size) + assertTrue(AudioSegmentation.pcm16leToFloat(events.pcmChunks[0]).all { it == 200 / 32768f }) + assertTrue(events.statuses.contains("Listening (cloud)...")) + + recorder.finish() + assertEquals(listOf("mix", "left temple", "right temple", "frames"), MkaReader(file.input()).tracks.map { it.name }) + val recorded = drain(MicRecordingReader(file.input())) + assertEquals(5, recorded.count { it.stream == MicStream.LEFT }) + assertEquals(5, recorded.count { it.stream == MicStream.RIGHT }) + val left = recorded.first { it.stream == MicStream.LEFT } + assertEquals(2, left.channels) + assertEquals(1000L, left.deviceTickMs) + assertEquals(20, left.angleDegrees) + } + + @Test + fun micArrayCaptureNeedsGlassesThatHaveTheChannel() { + val events = Events() + val session = session(FakeHost(), "voice-test-array-none") + session.setListener(events) + session.setMicArray(CONFIGURE_RECORD) + session.start("cloud", 1) + assertEquals>(listOf("Voice control needs an active G2 connection."), events.statuses) + // A record that is not a CONFIGURE leaves the capture on the mixed stream. + val mixed = Events() + session.setListener(mixed) + session.setMicArray(MicArrayProtocol.control(MicArrayProtocol.OP_STOP)) + session.setUsePhoneMic(true) + session.start("cloud", 2) + assertTrue(mixed.stoppedLatch.await(3000)) + assertEquals>(listOf("Could not start the phone microphone."), mixed.statuses) + } + + @Test + fun replayPlaysARecordingThroughThePipelineInRealTime() { + val host = FakeHost(recording = monoRecording(30)) + val events = Events() + val session = session(host, "voice-test-replay") + session.setListener(events) + // A replay is never recorded again. + val copy = MemoryFile() + val recorder = MicRecordingWriter(copy, null, 0, testPlatform()) + session.setRecorder(recorder) + session.setReplay("recording.mka", 1000) + val startedAt = testPlatform().elapsedRealtimeMs() + session.start("onboard", 5) + assertTrue(events.stoppedLatch.await(5000)) + val elapsed = testPlatform().elapsedRealtimeMs() - startedAt + // Half a second of audio, paced as it was captured. + assertTrue(elapsed >= 400, "replay took $elapsed ms") + assertEquals(10, events.pcmChunks.size) + assertEquals(1020.toShort(), AudioSegmentation.pcm16le(events.pcmChunks[0][0], events.pcmChunks[0][1])) + assertEquals("start" to 1000L, events.replay.first()) + assertEquals("end" to 1500L, events.replay.last()) + assertEquals(VoiceCaptureSession.REPLAY_FINISHED_STATUS, events.statuses.last()) + assertEquals("x" to true, events.transcripts.last()) + assertFalse(session.isCapturing()) + recorder.finish() + assertTrue(MkaReader(copy.input()).tracks.isEmpty()) + } + + @Test + fun replayJumpsAndPauses() { + val host = FakeHost(recording = monoRecording(200)) + val events = Events() + val session = session(host, "voice-test-replay-seek") + session.setListener(events) + session.setReplay("recording.mka", 0) + session.start("onboard", 6) + assertTrue(waitFor(3000) { events.pcmChunks.size >= 2 }) + session.setReplayPaused(true) + sleepMs(100) + val heldAt = events.pcmChunks.size + sleepMs(200) + assertEquals(heldAt, events.pcmChunks.size) + assertTrue(session.isCapturing()) + // Jump to the last quarter-second while paused, then go on. + session.seekReplay(9750) + session.setReplayPaused(false) + assertTrue(events.stoppedLatch.await(5000)) + assertTrue(events.replay.contains("seek" to 9750L), events.replay.toString()) + assertEquals("end" to 10_000L, events.replay.last()) + assertEquals(heldAt + 5, events.pcmChunks.size) + // What played before the jump ended as its own utterance. + assertEquals(2, events.transcripts.count { it.second }) + val last = events.pcmChunks.last() + assertEquals(1199.toShort(), AudioSegmentation.pcm16le(last[0], last[1])) + } + + @Test + fun replayOfAnUnreadableFileReportsAndStops() { + val events = Events() + val session = session(FakeHost(), "voice-test-replay-missing") + session.setListener(events) + session.setReplay("missing", 0) + session.start("cloud", 7) + assertTrue(events.stoppedLatch.await(3000)) + assertEquals>(listOf("Could not open the recording: no such file"), events.statuses) + assertEquals(listOf>("error" to 0L), events.replay) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt index 630d9bbc..c7b072e4 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt @@ -29,6 +29,8 @@ class VoiceCaptureSessionTest { override fun onFrameMeta(angleDegrees: Int, ssr: Int) { frameMeta.add(angleDegrees to ssr) } override fun onSpeechEnd() { speechEnds++ } override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) { verified = isWearer to similarity } + val replay = mutableListOf>() + override fun onReplay(event: String?, positionMs: Long, durationMs: Long) { replay.add(event to positionMs) } } /** Scripted phone mic: each entry is a chunk amplitude; after the script the read fails (-1). */ diff --git a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosMicRecorderTest.kt b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosMicRecorderTest.kt new file mode 100644 index 00000000..3b0a2d65 --- /dev/null +++ b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosMicRecorderTest.kt @@ -0,0 +1,46 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import platform.Foundation.NSTemporaryDirectory +import platform.Foundation.NSUUID + +class IosMicRecorderTest { + @Test + fun recordsPcmToAMatroskaFileThroughThePosixFileAdapters() { + val path = NSTemporaryDirectory() + NSUUID().UUIDString + ".mka" + try { + val recorder = assertNotNull(IosMicRecordings.open(path, "{\"platform\":\"ios\"}")) + // 2.5 s in 50 ms chunks: enough for whole clusters to reach the file before finish(). + for (index in 0 until 50) { + val samples = ShortArray(800) { (index * 10 + it % 7).toShort() } + recorder.append(AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size).data()) + } + assertFalse(recorder.failed) + val length = recorder.finish() + assertTrue(length >= 2500, "recorded $length ms") + assertEquals(fileLength(path), openFileInput(path).length) + + val reader = MicRecordingReader(openFileInput(path)) + assertEquals("{\"platform\":\"ios\"}", reader.configJson) + assertEquals("mic", reader.description) + var chunks = 0 + var last: MicChunk? = null + while (true) { + last = reader.next() ?: break + assertEquals(800, last.frames) + assertEquals((chunks * 10).toShort(), last.pcm[0]) + chunks++ + } + assertEquals(50, chunks) + reader.close() + } finally { + deleteFile(path) + } + assertNull(IosMicRecordings.open("/nonexistent-directory/x.mka", null)) + } +} diff --git a/tests/transcribe.test.cjs b/tests/transcribe.test.cjs index 1eef452c..7ee55802 100644 --- a/tests/transcribe.test.cjs +++ b/tests/transcribe.test.cjs @@ -4,6 +4,7 @@ const { loader } = require('./helpers/load-typescript.cjs'); const load = loader(); const { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts'); const { SpeechPauseDetector } = load('app/native/speech-pause.ts'); +const { formatReplayTime, stepReplaySeconds } = load('app/apps/transcribe/replay-time.ts'); const { TimedTranscript } = load('app/native/transcript-format.ts'); const { bordered, unbordered, edged } = require('./helpers/viewport.cjs'); @@ -128,3 +129,33 @@ test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores d.accept(pcm(3000)); assert.equal(d.accept(pcm(0, 1500)), true); }); + +test('a replay jump leaves its time on a line of its own and starts a new paragraph', () => { + const m = new TranscriptModel(); + m.mark('[0:30]'); + assert.equal(m.text, '[0:30]'); + m.accept({ text: 'After the jump', isFinal: false }); + assert.equal(m.text, '[0:30]\nAfter the jump'); + // What was still being spoken when the replay jumped is kept, ahead of the mark. + m.mark('[1:15]'); + m.accept({ text: 'Later.', isFinal: true }); + m.accept({ text: 'More.', isFinal: true }); + assert.equal(m.text, '[0:30]\nAfter the jump\n[1:15]\nLater. More.'); +}); + +test('replay times read as m:ss, with hours from an hour on', () => { + assert.equal(formatReplayTime(0), '0:00'); + assert.equal(formatReplayTime(83_999), '1:23'); + assert.equal(formatReplayTime(3_600_000), '1:00:00'); + assert.equal(formatReplayTime(83_000, true), '0:01:23'); + assert.equal(formatReplayTime(-5), '0:00'); +}); + +test('the time picker steps carry between fields and stay inside the recording', () => { + assert.equal(stepReplaySeconds(59, 'seconds', 1, 600), 60); + assert.equal(stepReplaySeconds(60, 'seconds', -1, 600), 59); + assert.equal(stepReplaySeconds(30, 'minutes', 2, 600), 150); + assert.equal(stepReplaySeconds(590, 'minutes', 1, 600), 600); + assert.equal(stepReplaySeconds(30, 'minutes', -1, 600), 0); + assert.equal(stepReplaySeconds(100, 'hours', 1, 7300), 3700); +}); diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index 3050d7c8..9c104cd5 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -164,6 +164,8 @@ function androidBridge(modules = {}) { setListener(listener) { this.listener = listener; }, setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {}, setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {}, + setMicArray() {}, setReplay() {}, setRecordingEnabled(enabled) { this.recordingEnabled = enabled; }, + noteRecordingEvent(json) { (this.recordingEvents ??= []).push(JSON.parse(json)); }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, stop() { this.active = false; }, isCapturing() { return this.active; }, @@ -217,24 +219,23 @@ test('a shared capture restarts for a holder that needs another provider, and is assert.equal(bridge.holdsCapture('ptt'), true); }); -test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => { +test('Android continuous captures record to the requested file, and report one that cannot be created', () => { const recordings = []; let fail = false, attempts = 0; - class FakeRecorder { - static open(name) { - attempts++; - if (fail) return null; - const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; - } - append(pcm) { this.pcm.push(...pcm); } - close() { this.closed = true; } - } const { bridge, controller } = androidBridge({ - './wav-recorder': { WavRecorder: FakeRecorder }, + './mic-recording': { newRecordingPath: name => `/Download/${name}.mka` }, './speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } }, '../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) }, '../util/audio-level': { pcmLevel: () => 0 }, }); + // The controller writes the file itself, from the capture's own thread. + controller.startRecording = (path, json) => { + attempts++; + if (fail) return false; + recordings.push({ path, info: JSON.parse(json), closed: false }); + return true; + }; + controller.stopRecording = () => { recordings.at(-1).closed = true; return 0; }; const options = { provider: 'onboard' }; let changes = 0; bridge.onRecordingChange(() => changes++); bridge.recordTo('transcribe-1'); @@ -242,20 +243,24 @@ test('Android continuous captures record to the requested WAV, and report one th // Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save. bridge.startPushToTalk(options); controller.listener.onPcm([9, 9]); - assert.equal(attempts, 0); + assert.deepEqual([attempts, controller.recordingEnabled], [0, false]); bridge.startContinuousCapture(options); controller.listener.onPcm([1, 2]); controller.listener.onPcm([3, 4]); - assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]); - // The file outlives a capture: one restarted after the glasses reconnect keeps appending to it. + assert.deepEqual(recordings.map(r => [r.path, r.info.recording, r.closed]), [['/Download/transcribe-1.mka', 'transcribe-1', false]]); + assert.deepEqual([attempts, controller.recordingEnabled], [1, true]); + // The file outlives a capture: one restarted after the glasses reconnect goes on in it, + // with the restart noted, and nothing is recorded while no capture holds the mic. bridge.handleSessionEnded(); + assert.equal(controller.recordingEnabled, false); bridge.resumeCapture(options); controller.listener.onPcm([5]); - assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]); + assert.deepEqual([attempts, recordings[0].closed, controller.recordingEnabled], [1, false, true]); + assert.deepEqual(controller.recordingEvents.map(event => [event.event, event.recording]), [['capture', 'transcribe-1']]); bridge.recordTo(''); assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]); controller.listener.onPcm([6]); - assert.equal(recordings[0].pcm.length, 5); + assert.equal(attempts, 1); // One attempt per recording, not one per packet; another recording is a fresh attempt. fail = true; attempts = 0; @@ -269,5 +274,5 @@ test('Android continuous captures record to the requested WAV, and report one th bridge.recordTo(''); bridge.recordTo('transcribe-2'); controller.listener.onPcm([10]); - assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]); + assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).path], [true, false, '/Download/transcribe-2.mka']); }); diff --git a/tests/voice-recording.test.cjs b/tests/voice-recording.test.cjs new file mode 100644 index 00000000..5255726a --- /dev/null +++ b/tests/voice-recording.test.cjs @@ -0,0 +1,209 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +/** The Android voice bridge over a fake native controller that records what it is told. */ +function harness({ recordingOk = true, downloads = '/downloads' } = {}) { + const controllers = []; + class FakeController { + constructor() { + this.calls = []; this.events = []; this.capturing = false; this.recording = null; + this.micArray = null; this.replay = null; this.recordingEnabled = null; + controllers.push(this); + // Setters the tests don't look at are only logged. + return new Proxy(this, { get: (target, name) => name in target ? target[name] : (...args) => { target.calls.push([name, ...args]); } }); + } + setListener(listener) { this.listener = listener; } + setMicArray(record) { this.micArray = record; } + setReplay(path, startMs) { this.replay = path === null ? null : { path, startMs }; } + start(mode) { this.calls.push(['start', mode]); this.capturing = true; } + stop() { this.calls.push(['stop']); this.capturing = false; } + isCapturing() { return this.capturing; } + startRecording(path, json) { if (recordingOk) this.recording = { path, info: JSON.parse(json) }; return recordingOk; } + setRecordingEnabled(enabled) { this.recordingEnabled = enabled; } + noteRecordingEvent(json) { this.events.push(JSON.parse(json)); } + stopRecording() { this.calls.push(['stopRecording']); this.recording = null; return 0; } + seekReplay(ms) { this.calls.push(['seekReplay', ms]); } + setReplayPaused(paused) { this.calls.push(['setReplayPaused', paused]); } + } + const module = loader({ + global: { isAndroid: true }, + com: { faceclaw: { app: { FaceclawVoiceController: FakeController, FaceclawVoiceControllerListener: function (impl) { return impl; } } } }, + }, { + '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, + './cloud-stt': { toJavaBytes: bytes => Array.from(bytes) }, + './cloud-stt-provider': { + createCloudSttClient: () => null, + transcriptionKey: options => `${options.provider}:${options.language ?? ''}`, + }, + './mic-recording': { newRecordingPath: name => downloads === null ? null : `${downloads}/${name}.mka` }, + '../util/array-util': { toUint8Array: value => value }, + })('app/native/voice-control.ts'); + const options = (extra = {}) => ({ + communicator: {}, provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '', ...extra, + }); + return { ...module, controllers, options, pcm: controller => controller.listener.onPcm(new Uint8Array(1600)) }; +} + +const CONFIGURE_2 = Uint8Array.from([0x4d, 0x43, 1, 1, 1, 1, 1, 0, 160, 0, 0, 0, 2]); +const CONFIGURE_4 = Uint8Array.from([0x4d, 0x43, 1, 1, 0, 3, 1, 0, 160, 0, 0, 0, 2]); + +test('a capture of the temples\' microphones arms them, and another microphone count restarts the stream', () => { + const h = harness(); + const bridge = h.voiceControlBridge; + bridge.startContinuousCapture(h.options({ micArray: { configure: CONFIGURE_2 } })); + const controller = h.controllers[0]; + assert.deepEqual(controller.micArray, Array.from(CONFIGURE_2)); + assert.equal(controller.replay, null); + assert.deepEqual(controller.calls.filter(call => call[0] === 'start'), [['start', 'onboard']]); + // Push-to-talk asks for the same thing while the window is open: it shares the stream. + bridge.startPushToTalk(h.options({ micArray: { configure: CONFIGURE_2 } })); + assert.equal(controller.calls.filter(call => call[0] === 'start').length, 1); + bridge.stopPushToTalk(); + // The Microphones setting changed. + bridge.retargetContinuousCapture(h.options({ micArray: { configure: CONFIGURE_4 } })); + assert.deepEqual(controller.micArray, Array.from(CONFIGURE_4)); + assert.deepEqual(controller.calls.filter(call => call[0] === 'start' || call[0] === 'stop').map(call => call[0]), ['start', 'stop', 'start']); + // Back to the mixed stream; the phone's microphone never gets an array. + bridge.retargetContinuousCapture(h.options()); + assert.equal(controller.micArray, null); + bridge.retargetContinuousCapture(h.options({ communicator: null, usePhoneMic: true, micArray: { configure: CONFIGURE_4 } })); + assert.equal(controller.micArray, null); +}); + +test('a recording opens with the first audio, describes its capture, and follows the capture across restarts', () => { + const h = harness(); + const bridge = h.voiceControlBridge; + const changes = []; + bridge.onRecordingChange(() => changes.push(bridge.recording)); + bridge.recordTo('transcribe-1'); + assert.equal(bridge.recording, true); + bridge.startContinuousCapture(h.options({ micArray: { configure: CONFIGURE_4 }, noiseSuppression: true, captureInfo: { mics: 4, glasses: 'G2' } })); + const controller = h.controllers[0]; + // Nothing on disk until audio arrives: a capture that never starts leaves no file. + assert.equal(controller.recording, null); + assert.equal(controller.recordingEnabled, true); + h.pcm(controller); + assert.equal(controller.recording.path, '/downloads/transcribe-1.mka'); + const info = controller.recording.info; + assert.equal(info.recording, 'transcribe-1'); + assert.equal(info.source, 'glasses'); + assert.equal(info.micArray, true); + assert.equal(info.mics, 4); + assert.equal(info.glasses, 'G2'); + assert.deepEqual(info.transcription, { provider: 'onboard', language: '' }); + assert.deepEqual(info.processing, { noiseSuppression: true, beamFilter: null }); + h.pcm(controller); + assert.equal(controller.calls.filter(call => call[0] === 'stopRecording').length, 0); + + // The capture restarts with other microphones: the same file goes on, with the change noted in it. + bridge.retargetContinuousCapture(h.options({ captureInfo: { mics: 1, glasses: 'G2' } })); + assert.equal(controller.events.length, 1); + assert.equal(controller.events[0].event, 'capture'); + assert.equal(controller.events[0].mics, 1); + assert.equal(controller.events[0].micArray, false); + assert.equal(controller.recording.path, '/downloads/transcribe-1.mka'); + + // Save Recording was turned off. + bridge.recordTo(''); + assert.equal(bridge.recording, false); + assert.equal(controller.recording, null); + assert.deepEqual(changes, [true, false]); +}); + +test('push-to-talk on a capture of its own is not recorded', () => { + const h = harness(); + const bridge = h.voiceControlBridge; + bridge.recordTo('transcribe-1'); + bridge.startPushToTalk(h.options()); + const controller = h.controllers[0]; + assert.equal(controller.recordingEnabled, false); + h.pcm(controller); + assert.equal(controller.recording, null); + // The Transcribe window joins the running stream: from here it is recorded. + bridge.startContinuousCapture(h.options()); + assert.equal(controller.recordingEnabled, true); + h.pcm(controller); + assert.equal(controller.recording.path, '/downloads/transcribe-1.mka'); + bridge.stopContinuousCapture(); + assert.equal(controller.recordingEnabled, false); +}); + +test('a recording that cannot be created is reported once', () => { + for (const h of [harness({ recordingOk: false }), harness({ downloads: null })]) { + const bridge = h.voiceControlBridge; + bridge.recordTo('transcribe-1'); + bridge.startContinuousCapture(h.options()); + h.pcm(h.controllers[0]); + assert.equal(bridge.recording, false); + assert.equal(bridge.recordingFailed, true); + } +}); + +test('a replay runs on its own controller, follows its events, jumps while playing and restarts after the end', () => { + const h = harness(); + const replay = new h.VoiceReplay('/downloads/talk.mka', () => h.options({ communicator: {}, micArray: { configure: CONFIGURE_4 } })); + const seen = []; + replay.onChange(jumpedToMs => seen.push([replay.state, replay.positionMs, jumpedToMs])); + assert.equal(h.controllers.length, 0); + replay.start(); + // The microphone's bridge has not been touched. + assert.equal(h.controllers.length, 1); + const controller = h.controllers[0]; + assert.deepEqual(controller.replay, { path: '/downloads/talk.mka', startMs: 0 }); + assert.equal(controller.micArray, null); + assert.equal(replay.state, 'opening'); + const statuses = []; + replay.events.onStatus(state => statuses.push(state.status)); + assert.equal(statuses.at(-1), 'Opening the recording...'); + + controller.listener.onReplay('start', 0, 60_000); + assert.equal(replay.state, 'playing'); + assert.equal(replay.durationMs, 60_000); + controller.listener.onReplay('position', 1000, 60_000); + assert.equal(replay.positionMs, 1000); + // A plain start and the ticking position are not jumps. + assert.deepEqual(seen.map(entry => entry[2]), [undefined, undefined, undefined]); + + replay.setPaused(true); + assert.equal(replay.state, 'paused'); + replay.setPaused(false); + replay.jumpTo(90_000); // past the end: held to the recording's length + assert.deepEqual(controller.calls.filter(call => call[0] === 'seekReplay' || call[0] === 'setReplayPaused'), + [['setReplayPaused', true], ['setReplayPaused', false], ['seekReplay', 60_000]]); + replay.jumpTo(30_000); + controller.listener.onReplay('seek', 30_000, 60_000); + assert.deepEqual(seen.at(-1), ['playing', 30_000, 30_000]); + + controller.listener.onReplay('end', 60_000, 60_000); + controller.capturing = false; + assert.equal(replay.state, 'ended'); + assert.equal(replay.positionMs, 60_000); + replay.setPaused(true); // nothing to pause + assert.equal(replay.state, 'ended'); + + // A jump once it has ended plays again from there. + replay.jumpTo(45_000); + assert.deepEqual(controller.replay, { path: '/downloads/talk.mka', startMs: 45_000 }); + assert.equal(replay.state, 'opening'); + controller.listener.onReplay('start', 45_000, 60_000); + assert.deepEqual(seen.at(-1), ['playing', 45_000, 45_000]); + + replay.close(); + assert.equal(controller.calls.at(-1)[0], 'stop'); + replay.start(); + assert.equal(controller.calls.at(-1)[0], 'stop'); +}); + +test('a recording that cannot be opened fails the replay', () => { + const h = harness(); + const replay = new h.VoiceReplay('/downloads/missing.mka', () => h.options()); + replay.start(); + const controller = h.controllers[0]; + controller.listener.onStatus('Could not open the recording: no such file'); + controller.listener.onReplay('error', 0, 0); + assert.equal(replay.state, 'failed'); + const statuses = []; + replay.events.onStatus(state => statuses.push(state.status)); + assert.deepEqual(statuses, ['Could not open the recording: no such file']); +}); diff --git a/tests/wav-header.test.cjs b/tests/wav-header.test.cjs deleted file mode 100644 index ce2744a6..00000000 --- a/tests/wav-header.test.cjs +++ /dev/null @@ -1,19 +0,0 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const { loader } = require('./helpers/load-typescript.cjs'); - -test('the WAV header describes 16 kHz mono PCM16 and its data size', () => { - const { wavHeader } = loader({ DataView }, {})('app/native/wav-header.ts'); - const header = Buffer.from(wavHeader(32000)); - assert.equal(header.length, 44); - assert.equal(header.toString('ascii', 0, 4), 'RIFF'); - assert.equal(header.readUInt32LE(4), 36 + 32000); - assert.equal(header.toString('ascii', 8, 16), 'WAVEfmt '); - assert.equal(header.readUInt16LE(20), 1); - assert.equal(header.readUInt16LE(22), 1); - assert.equal(header.readUInt32LE(24), 16000); - assert.equal(header.readUInt32LE(28), 32000); - assert.equal(header.readUInt16LE(34), 16); - assert.equal(header.toString('ascii', 36, 40), 'data'); - assert.equal(header.readUInt32LE(40), 32000); -});