Multi-mic support and replay mode for Transcribe app
This commit is contained in:
1 parent
c162c8e633
commit
9372b2742c
45 files changed
+3926
-288
No files matched your search
@@ -114,6 +114,21 @@ class FaceclawVoiceController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
override fun micArrayLink(): MicArrayLink? {
|
||||
val currentCommunicator = communicator ?: return null
|
||||
return object : MicArrayLink {
|
||||
override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean =
|
||||
currentCommunicator.startG2AudioForwarding(listener)
|
||||
|
||||
override fun stopForwarding() = currentCommunicator.stopG2AudioForwarding()
|
||||
|
||||
override fun isForwarding(): Boolean = currentCommunicator.isAudioCaptureActive()
|
||||
|
||||
override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
currentCommunicator.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder =
|
||||
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
|
||||
else AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
@@ -140,6 +155,11 @@ class FaceclawVoiceController(context: Context) {
|
||||
|
||||
private val session = VoiceCaptureSession(host)
|
||||
|
||||
// The recording of this controller's live captures (see startRecording). It outlives any one
|
||||
// capture, and is only handed to the session while recording is enabled.
|
||||
private var recording: MicRecordingWriter? = null
|
||||
private var recordingEnabled = true
|
||||
|
||||
fun setListener(listener: FaceclawVoiceControllerListener?) {
|
||||
session.setListener(listener)
|
||||
}
|
||||
@@ -201,6 +221,88 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setBeamFilter(enabled, centerDeg, halfWidthDeg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture glasses sessions from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the stock mixed stream. [configureRecord] is the CONFIGURE
|
||||
* record to arm both temples with; null goes back to the mixed stream. Must be set before
|
||||
* {@link #start}.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
session.setMicArray(configureRecord)
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next {@link #start} play the recording at [path] (a .mka file) from [startMs] in
|
||||
* place of any microphone; null goes back to the microphones. Progress arrives through the
|
||||
* listener's onReplay.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
session.setReplay(path, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
session.seekReplay(positionMs)
|
||||
}
|
||||
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
session.setReplayPaused(paused)
|
||||
}
|
||||
|
||||
/**
|
||||
* Record live captures to a new Matroska audio file at [path], from now until
|
||||
* {@link #stopRecording}, across however many captures run in between: every source's audio
|
||||
* as it arrives, before any processing. [configJson] describes the capture and is stored in
|
||||
* the file. Returns false when the file cannot be created.
|
||||
*/
|
||||
@Synchronized
|
||||
fun startRecording(path: String, configJson: String?): Boolean {
|
||||
stopRecording()
|
||||
return try {
|
||||
val parent = File(path).parentFile
|
||||
if (parent != null && !parent.exists()) parent.mkdirs()
|
||||
recording = MicRecordingWriter(openFileOutput(path), configJson, System.currentTimeMillis())
|
||||
session.setRecorder(if (recordingEnabled) recording else null)
|
||||
true
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "could not create recording $path", t)
|
||||
recording = null
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether captures are written to the recording right now: off while the capture running
|
||||
* isn't one that is to be recorded.
|
||||
*/
|
||||
@Synchronized
|
||||
fun setRecordingEnabled(enabled: Boolean) {
|
||||
recordingEnabled = enabled
|
||||
session.setRecorder(if (enabled) recording else null)
|
||||
}
|
||||
|
||||
/** Whether a recording is open and still being written (false once writing to it failed). */
|
||||
@Synchronized
|
||||
fun isRecording(): Boolean = recording?.failed == false
|
||||
|
||||
/**
|
||||
* Note a change in what is being captured (a JSON object) at the recording's current
|
||||
* position; the audio after it is placed afresh on the recording's timeline.
|
||||
*/
|
||||
@Synchronized
|
||||
fun noteRecordingEvent(json: String) {
|
||||
recording?.event(json)
|
||||
}
|
||||
|
||||
/** Complete the recording's file. Returns its length in ms, or -1 when none was open. */
|
||||
@Synchronized
|
||||
fun stopRecording(): Long {
|
||||
val current = recording ?: return -1
|
||||
recording = null
|
||||
session.setRecorder(null)
|
||||
return current.finish()
|
||||
}
|
||||
|
||||
fun start(requestedMode: String?) {
|
||||
session.start(requestedMode, 0)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { isMicRecordingFile } from "../../native/mic-recording";
|
||||
import { isPlayableVideoFile } from "../../native/video-player";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
openFontWindow: (title: string, path: string) => void;
|
||||
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||
openVideoWindow: (title: string, path: string) => void;
|
||||
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
|
||||
openRecordingInTranscribe: (title: string, path: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) =>
|
||||
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isMicRecordingFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
label: "Open in Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openRecordingInTranscribe(entry.name, entry.path);
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isPlayableVideoFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
FILES_WINDOW_ID,
|
||||
} from "./files-app";
|
||||
import { createVideoPlayerWindow } from "./video-window";
|
||||
import { openRecordingInTranscribe } from "../transcribe";
|
||||
|
||||
// Document windows are closeable and non-singleton; the serial keeps their
|
||||
// windowIds unique.
|
||||
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
|
||||
},
|
||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||
openRecordingInTranscribe: (title, path) => {
|
||||
openRecordingInTranscribe(ctx, path, title);
|
||||
},
|
||||
}),
|
||||
),
|
||||
// A document arriving via Android's Share intent opens as its own window.
|
||||
|
||||
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration a capture of the temples' own microphones arms them
|
||||
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
|
||||
* of each. The front end and direction detection follow Microphone setup
|
||||
* (the Microphones app), except that both of a temple's microphones only
|
||||
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
|
||||
* what the firmware streams and transcription takes.
|
||||
*/
|
||||
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
|
||||
const base = loadMicConfig();
|
||||
return {
|
||||
...base,
|
||||
source: mics === 4 ? "codec" : base.source,
|
||||
channelMask: mics === 4 ? 0b11 : 0b01,
|
||||
codec: "raw",
|
||||
format: "pcm16",
|
||||
sampleRateHz: 16_000,
|
||||
armHardware: true,
|
||||
};
|
||||
}
|
||||
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
|
||||
@@ -1,5 +1,32 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
|
||||
import { type AppContext, type AppDefinition } from "../app-definition";
|
||||
import {
|
||||
createTranscribeAppWindow,
|
||||
createTranscribeReplayWindow,
|
||||
TRANSCRIBE_SURFACE_ID,
|
||||
TRANSCRIBE_WINDOW_ID,
|
||||
} from "./transcribe-app";
|
||||
|
||||
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
|
||||
let nextReplaySerial = 1;
|
||||
|
||||
/**
|
||||
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
|
||||
* own. Returns false where recordings can't be replayed.
|
||||
*/
|
||||
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
|
||||
if (!ctx.actions.openVoiceReplay) return false;
|
||||
const windowId = `transcribe:replay:${nextReplaySerial++}`;
|
||||
void ctx
|
||||
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
|
||||
const window = createTranscribeReplayWindow(windowId, path, name, options);
|
||||
if (!window) throw new Error("recordings can't be replayed here");
|
||||
return window;
|
||||
})
|
||||
.catch((error) => {
|
||||
ctx.appendLog(`transcribe replay window failed: ${error}`);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const transcribeApp: AppDefinition = {
|
||||
appId: "transcribe",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
|
||||
export function formatReplayTime(ms: number, withHours = false): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const seconds = String(total % 60).padStart(2, "0");
|
||||
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
|
||||
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
|
||||
}
|
||||
|
||||
/** The time picker's fields, in display and click order. */
|
||||
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
|
||||
export type TimeField = (typeof TIME_FIELDS)[number];
|
||||
|
||||
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
|
||||
|
||||
/**
|
||||
* Step a position (whole seconds) by `delta` of a field's unit, carrying
|
||||
* into the neighboring fields, within [0, maxSeconds].
|
||||
*/
|
||||
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
|
||||
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
|
||||
import { CONTEXT_MENU_DIM } from "../../ui/menu";
|
||||
import { centeredTextY } from "../../ui/metrics";
|
||||
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
|
||||
|
||||
const PADDING = 12;
|
||||
const FIELD_GAP = 6;
|
||||
const FIELD_PAD_X = 6;
|
||||
/** Fill and outline of the active field (the Timers and date pickers' field style). */
|
||||
const ACTIVE_FILL = 35;
|
||||
const ACTIVE_STROKE = 130;
|
||||
|
||||
export type TimePickerOptions = {
|
||||
title: string;
|
||||
/** Where the picker starts, and the furthest it goes, in ms. */
|
||||
initialMs: number;
|
||||
maxMs: number;
|
||||
onPick: (positionMs: number) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* A modal position picker for a recording: hours (when the recording has
|
||||
* any), minutes and seconds, over the dimmed content beneath. Input follows
|
||||
* the date picker's: scroll adjusts the active field (up is later, carrying
|
||||
* into the fields beside it), click moves to the next field and on the last
|
||||
* one picks, double-click steps back a field and on the first one cancels.
|
||||
* A watch's swipes move between fields and adjust.
|
||||
*/
|
||||
export class TimePickerLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
readonly dimUnderneath = CONTEXT_MENU_DIM;
|
||||
private readonly fields: readonly TimeField[];
|
||||
private readonly maxSeconds: number;
|
||||
private seconds: number;
|
||||
private field = 0;
|
||||
|
||||
constructor(private readonly options: TimePickerOptions) {
|
||||
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
|
||||
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
|
||||
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "swipe-up":
|
||||
this.adjust(1);
|
||||
return;
|
||||
case "scroll-down":
|
||||
case "swipe-down":
|
||||
this.adjust(-1);
|
||||
return;
|
||||
case "swipe-right":
|
||||
this.field = Math.min(this.fields.length - 1, this.field + 1);
|
||||
return;
|
||||
case "swipe-left":
|
||||
this.field = Math.max(0, this.field - 1);
|
||||
return;
|
||||
case "click":
|
||||
if (this.field < this.fields.length - 1) {
|
||||
this.field++;
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
this.options.onPick(this.seconds * 1000);
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
if (this.field > 0) this.field--;
|
||||
else ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private adjust(delta: number): void {
|
||||
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
|
||||
}
|
||||
|
||||
private hints(field: number): string {
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, "adjust"],
|
||||
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
|
||||
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
|
||||
]);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const image = paintBelow();
|
||||
const small = getDefaultSmallFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const focused = ctx.stack.isFocused();
|
||||
const base = ctx.stack.getBaseSize();
|
||||
|
||||
const values = [
|
||||
String(Math.floor(this.seconds / 3600)),
|
||||
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
|
||||
String(this.seconds % 60).padStart(2, "0"),
|
||||
].slice(TIME_FIELDS.length - this.fields.length);
|
||||
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
|
||||
const fieldWidths = this.fields.map(
|
||||
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
|
||||
);
|
||||
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
|
||||
const boxHeight = medium.lineHeight + 6;
|
||||
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
|
||||
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
|
||||
const width = Math.min(
|
||||
base.width,
|
||||
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
|
||||
);
|
||||
const titleHeight = small.lineHeight + 12;
|
||||
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
|
||||
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
|
||||
const x = Math.max(0, Math.floor((base.width - width) / 2));
|
||||
const y = Math.max(0, Math.floor((base.height - height) / 2));
|
||||
|
||||
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
|
||||
image.fillRoundedRect(x, y, width, height, 1);
|
||||
image.drawRoundedRect(x, y, width, height, 72);
|
||||
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
|
||||
|
||||
const top = y + 8 + titleHeight;
|
||||
let fieldX = x + PADDING;
|
||||
for (let index = 0; index < this.fields.length; index++) {
|
||||
const fieldWidth = fieldWidths[index]!;
|
||||
const active = index === this.field;
|
||||
if (active) {
|
||||
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
|
||||
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
|
||||
}
|
||||
const value = values[index]!;
|
||||
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
|
||||
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
|
||||
const caption = this.fields[index]!;
|
||||
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
|
||||
top + boxHeight + 2, caption, active ? 190 : 110);
|
||||
fieldX += fieldWidth + FIELD_GAP;
|
||||
}
|
||||
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
|
||||
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
import { withRecordingIcon } from "../../ui/listening-header";
|
||||
import { TimePickerLayer } from "./time-picker";
|
||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
|
||||
}
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
// One timestamp per session names both the audio recording and a saved transcript.
|
||||
const stamp = transcriptTimestamp();
|
||||
const layer = new TranscribeLayer(stamp);
|
||||
// At launch, and again when the Microphones setting changes: a running
|
||||
// capture moves over to the microphones now asked for.
|
||||
const startCapture = () =>
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
},
|
||||
// The controller starts and stops the recording as the setting changes.
|
||||
toggleSettingMenuItem(saveRecordingSetting),
|
||||
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
app.requestRender();
|
||||
});
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
startCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
return app;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe in replay mode: a window of its own that transcribes a
|
||||
* recording (a .mka file from Save Recording) in place of the microphones,
|
||||
* in real time and through the same pipeline. It neither uses the mic nor
|
||||
* shows the mic tray icon, so it runs beside a live Transcribe window. The
|
||||
* header shows the position in the recording; a tap pauses and resumes; the
|
||||
* menu jumps to a time.
|
||||
*/
|
||||
export function createTranscribeReplayWindow(
|
||||
windowId: string,
|
||||
path: string,
|
||||
name: string,
|
||||
options: InProcessAppOptions,
|
||||
): InProcessWindow | null {
|
||||
const replay = options.actions.openVoiceReplay?.(path) ?? null;
|
||||
if (!replay) return null;
|
||||
// Names a saved transcript after the recording it came from.
|
||||
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId,
|
||||
title: name,
|
||||
iconLetter: "Tr",
|
||||
icon: "mic",
|
||||
closeable: true,
|
||||
// Relaunching Transcribe after a restart would open the microphone, not this recording.
|
||||
restoresApp: false,
|
||||
// Like a playing video: something to watch, not to touch.
|
||||
keepsScreenOn: () => replay.state === "playing",
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [
|
||||
{
|
||||
label: "Jump to time",
|
||||
// The length isn't known until the recording has been opened.
|
||||
disabled: () => replay.durationMs <= 0,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(
|
||||
new TimePickerLayer({
|
||||
title: "Jump to time",
|
||||
initialMs: replay.positionMs,
|
||||
maxMs: replay.durationMs,
|
||||
onPick: (positionMs) => {
|
||||
replay.jumpTo(positionMs);
|
||||
app.requestRender();
|
||||
},
|
||||
}),
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: replay.state === "paused" ? "Resume" : "Pause",
|
||||
disabled: () => replay.state !== "playing" && replay.state !== "paused",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.setPaused(replay.state === "playing");
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Restart from 0:00",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.jumpTo(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
},
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
replay.close();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// Started first, so the layer picks its opening status up as it subscribes.
|
||||
replay.start();
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
|
||||
import { type MicCount } from "../../ui/layers";
|
||||
import { micControlSupported } from "../microphones/mic-control";
|
||||
|
||||
export type TranscribeMics = "1" | "2" | "4";
|
||||
|
||||
const MICS_LABELS: Record<TranscribeMics, string> = {
|
||||
"1": "1 (mixed)",
|
||||
"2": "2 (one per temple)",
|
||||
"4": "4 (two per temple)",
|
||||
};
|
||||
|
||||
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
|
||||
export function separateMicsSupported(): boolean {
|
||||
return glassesTypeSetting.get() === "g2" && micControlSupported();
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones Transcribe captures. With more than
|
||||
* one, the temples stream their own microphones (each temple on its own
|
||||
* clock) instead of the glasses' mixed stream; a recording keeps every one,
|
||||
* and what is transcribed is a mix made on the phone.
|
||||
*/
|
||||
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
|
||||
id: "transcribe-mics",
|
||||
label: "Microphones",
|
||||
storageKey: "transcribe.mics",
|
||||
defaultValue: "1",
|
||||
values: ["1", "2", "4"],
|
||||
formatValue: (value) => MICS_LABELS[value],
|
||||
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
|
||||
description:
|
||||
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
|
||||
});
|
||||
|
||||
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
|
||||
export function transcribeMicCount(): MicCount {
|
||||
if (!separateMicsSupported()) return 1;
|
||||
return Number(transcribeMicsSetting.get()) as MicCount;
|
||||
}
|
||||
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { formatReplayTime } from "./replay-time";
|
||||
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*
|
||||
* Given a replay, it transcribes that recording instead of the microphone:
|
||||
* the header shows the position in the recording, a tap pauses and resumes,
|
||||
* and each jump leaves its time as a line in the transcript.
|
||||
*/
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeReplay: (() => void) | null = null;
|
||||
private readonly source: VoiceEventSource;
|
||||
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
/**
|
||||
* `stamp` names the saved transcript, matching the session's audio
|
||||
* recording. `replay` is the recording to transcribe in place of the
|
||||
* microphone; `replayName` is what to call it.
|
||||
*/
|
||||
constructor(
|
||||
private readonly stamp: string = transcriptTimestamp(),
|
||||
private readonly replay: VoiceReplay | null = null,
|
||||
private readonly replayName = "",
|
||||
) {
|
||||
this.source = replay ? replay.events : voiceControlBridge;
|
||||
if (replay) this.status = "Opening the recording...";
|
||||
}
|
||||
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.meter.start(requestRender, isVisible, this.source);
|
||||
this.unsubscribeTranscript = this.source.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
|
||||
// The current status arrives while subscribing, before the window has a
|
||||
// surface to render to; the window paints it when it is first shown.
|
||||
let subscribing = true;
|
||||
this.unsubscribeStatus = this.source.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
if (!subscribing) requestRender();
|
||||
});
|
||||
subscribing = false;
|
||||
const replay = this.replay;
|
||||
if (replay) {
|
||||
let shownSecond = -1;
|
||||
let shownState = replay.state;
|
||||
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
|
||||
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
|
||||
// The clock ticks in whole seconds; repaint when what it shows changed.
|
||||
const second = Math.floor(replay.positionMs / 1000);
|
||||
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
|
||||
shownSecond = second;
|
||||
shownState = replay.state;
|
||||
if (isVisible()) requestRender();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
|
||||
private replayStatus(replay: VoiceReplay): string {
|
||||
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
|
||||
const withHours = replay.durationMs >= 3_600_000;
|
||||
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
|
||||
if (replay.state === "paused") return `${clock} · Paused`;
|
||||
if (replay.state === "ended") return `${clock} · Finished`;
|
||||
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
|
||||
return this.listening ? clock : `${clock} · ${this.status}`;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
|
||||
const viewport = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(viewport.width, viewport.height, 0);
|
||||
|
||||
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const replay = this.replay;
|
||||
const status = replay
|
||||
? this.replayStatus(replay)
|
||||
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const playing = !replay || replay.state === "playing";
|
||||
const bodyTop = paintListeningHeader(
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
|
||||
voiceControlBridge.recording,
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
|
||||
this.source.recording,
|
||||
);
|
||||
const step = lineStep(font) + 2;
|
||||
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
|
||||
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
|
||||
|
||||
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
|
||||
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click" && this.replay) {
|
||||
this.replay.setPaused(this.replay.state === "playing");
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribeReplay?.();
|
||||
this.unsubscribeReplay = null;
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -36,6 +36,18 @@ export class TranscriptModel {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* A line of its own (where a replay jumped to), ending whatever was in
|
||||
* progress: what follows it starts a new paragraph.
|
||||
*/
|
||||
mark(label: string): void {
|
||||
const before = this.text;
|
||||
this.finalized = before ? `${before}\n${label}` : label;
|
||||
this.live = "";
|
||||
this.separator = "\n";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
|
||||
@@ -53,7 +53,7 @@ import { type GlanceEvent } from "./glance-state";
|
||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { VoiceReplay, voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceActivity } from "../ui/shell/voice-activity";
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
@@ -76,7 +76,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { sameEdges } from "../ui/viewport";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
@@ -98,6 +98,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
|
||||
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
|
||||
import { wearerVerificationOptions } from "../apps/microphones/speakers";
|
||||
import { micSession } from "../apps/microphones/mic-session";
|
||||
import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control";
|
||||
import { encodeMicControl } from "../apps/microphones/mic-protocol";
|
||||
import {
|
||||
chargingPreviewMessage,
|
||||
glassesDisplayAsleep,
|
||||
@@ -410,6 +412,7 @@ class DashboardController {
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
|
||||
this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
|
||||
openVoiceReplay: (path: string) => this.openVoiceReplay(path),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
@@ -2253,6 +2256,30 @@ class DashboardController {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones the continuous-capture windows call
|
||||
* for (the most any asks for), where the glasses can stream them
|
||||
* separately: G2 with the microphone channel in use. Otherwise 1, the
|
||||
* mixed stream.
|
||||
*/
|
||||
private continuousMics(): MicCount {
|
||||
if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1;
|
||||
let mics: MicCount = 1;
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if ((options.mics ?? 1) > mics) mics = options.mics!;
|
||||
}
|
||||
return mics;
|
||||
}
|
||||
|
||||
/**
|
||||
* A recording as a capture of its own (see VoiceReplay), transcribed with
|
||||
* the provider and processing settings a microphone capture would get.
|
||||
* "My voice only" is left out: a recording is of whoever spoke.
|
||||
*/
|
||||
private openVoiceReplay(path: string): VoiceReplay {
|
||||
return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous capture's claim on the mic (the Microphones app and EvenHub
|
||||
* mic apps stand down while it is held) follows whether the capture really
|
||||
@@ -2278,9 +2305,26 @@ class DashboardController {
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
// Separate microphones only come from connected glasses.
|
||||
const mics = communicator ? this.continuousMics() : 1;
|
||||
const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics);
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null,
|
||||
// Stored in a recording of the capture (see Save Recording).
|
||||
captureInfo: {
|
||||
glasses: communicator ? glassesModelName() : null,
|
||||
mics,
|
||||
micConfig: micConfig && {
|
||||
source: micConfig.source,
|
||||
channelMask: micConfig.channelMask,
|
||||
codec: micConfig.codec,
|
||||
format: micConfig.format,
|
||||
sampleRateHz: micConfig.sampleRateHz,
|
||||
directionDetection: micConfig.beamform,
|
||||
},
|
||||
},
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
@@ -2721,10 +2765,11 @@ class DashboardController {
|
||||
if (!this.openAppsRestored || this.suppressOpenAppsPersist) return;
|
||||
const open: string[] = [];
|
||||
for (const window of shell.getWindows()) {
|
||||
if (window.appId === "launcher") continue;
|
||||
if (window.appId === "launcher" || window.restoresApp === false) continue;
|
||||
if (!open.includes(window.appId)) open.push(window.appId);
|
||||
}
|
||||
savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null });
|
||||
const foreground = shell.foregroundWindow();
|
||||
savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null });
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1111,8 +1111,8 @@ export class IosPreviewController {
|
||||
/**
|
||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||
* Recording is on.
|
||||
* on Android. `recordAs` names a recording (.mka) of the audio in Downloads
|
||||
* when Save Recording is on.
|
||||
*/
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
|
||||
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
|
||||
declare const FaceclawKitIosMicRecordings: any
|
||||
|
||||
/**
|
||||
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
|
||||
* Documents/Downloads while it is captured, through the shared Kotlin writer
|
||||
* that Android's captures are recorded with. The file is written about a
|
||||
* second at a time, so a crash leaves everything up to the last second.
|
||||
*/
|
||||
export class MicRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/**
|
||||
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
|
||||
* `info` describes the capture and is stored in the file.
|
||||
*/
|
||||
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.mka`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
|
||||
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
|
||||
return recorder ? new MicRecorder(recorder, path) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.recorder || !pcm.length) return
|
||||
// The handle is the buffer's start: a view into a larger buffer is copied out first.
|
||||
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
|
||||
try {
|
||||
// Copied by the recorder before this returns.
|
||||
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
this.close()
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.recorder?.finish() } catch { /* already closed */ }
|
||||
this.recorder = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
|
||||
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
|
||||
export const RECORDING_EXTENSION = ".mka";
|
||||
|
||||
export function isMicRecordingFile(name: string): boolean {
|
||||
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
|
||||
}
|
||||
|
||||
/**
|
||||
* The path for a new recording in the public Downloads directory:
|
||||
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
|
||||
* Downloads isn't available. The file itself is created by whoever records.
|
||||
*/
|
||||
export function newRecordingPath(name: string): string | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
|
||||
return String(file.getAbsolutePath());
|
||||
} catch (error) {
|
||||
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { MicRecorder } from './mic-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
|
||||
private continuous = false
|
||||
// Shown instead of the native "ended" status once a drain the bridge started completes.
|
||||
private drainStatus: string | null = null
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
private recorder: MicRecorder | null = null
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false
|
||||
private readonly recordingListeners = new Set<() => void>()
|
||||
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or ''. Only cloud captures supply PCM.
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
|
||||
private record(pcm: Uint8Array): void {
|
||||
if (!this.recording) return
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName)
|
||||
this.recorder = MicRecorder.open(this.recordingName, {
|
||||
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
|
||||
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
|
||||
})
|
||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||
}
|
||||
|
||||
+319
-27
@@ -1,9 +1,9 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { newRecordingPath } from "./mic-recording";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { WavRecorder } from "./wav-recorder";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
@@ -84,6 +84,49 @@ export type PushToTalkOptions = {
|
||||
* frame). Null/absent leaves the mic omnidirectional.
|
||||
*/
|
||||
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
|
||||
/**
|
||||
* Capture from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the glasses' mixed stream. Each temple
|
||||
* streams its microphones on its own link and clock; the phone mixes them
|
||||
* into the mono feed that is transcribed. Null/absent keeps the mixed
|
||||
* stream; ignored with usePhoneMic.
|
||||
*/
|
||||
micArray?: MicArrayCapture | null;
|
||||
/**
|
||||
* Play this recording (a .mka file from Save Recording) in real time in
|
||||
* place of any microphone. The capture needs no glasses, and ends by
|
||||
* itself when the recording does.
|
||||
*/
|
||||
replay?: { path: string; startMs: number } | null;
|
||||
/** What the caller knows of the capture, for the description stored in a recording of it. */
|
||||
captureInfo?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type MicArrayCapture = {
|
||||
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
|
||||
configure: Uint8Array;
|
||||
};
|
||||
|
||||
/** Progress of a capture that replays a recording; positions are ms into it. */
|
||||
export type VoiceReplayEvent = {
|
||||
/**
|
||||
* `start` once the file is open, `position` about once per second played,
|
||||
* `seek` when a jump took effect, `end` when the recording ran out, and
|
||||
* `error` when it could not be opened (the status says why).
|
||||
*/
|
||||
kind: "start" | "position" | "seek" | "end" | "error";
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
|
||||
export type VoiceEventSource = {
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void;
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
|
||||
onSpeechPause(listener: () => void): () => void;
|
||||
onMicLevel(listener: (level: number) => void): () => void;
|
||||
readonly recording: boolean;
|
||||
readonly recordingFailed: boolean;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
@@ -130,9 +173,13 @@ export class FaceclawVoiceControlBridge {
|
||||
private captureEndpointing = false;
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// What the running STT stream transcribes with (transcriptionKey); a holder
|
||||
// asking for something else restarts the stream rather than sharing it.
|
||||
// What the running STT stream captures from and transcribes with
|
||||
// (streamKey); a holder asking for something else restarts the stream
|
||||
// rather than sharing it.
|
||||
private streamProvider: string | null = null;
|
||||
// The options of the running stream, for the description of a recording of it.
|
||||
private streamOptions: PushToTalkOptions | null = null;
|
||||
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
|
||||
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
|
||||
// controller runs in the decode-only "cloud" mode and every decoded PCM
|
||||
// frame is broadcast to these listeners. STT capture preempts it — a live
|
||||
@@ -150,11 +197,14 @@ export class FaceclawVoiceControlBridge {
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set: every source's chunks as they arrive, before any processing, written
|
||||
// by the controller. The file outlives a capture, so a restarted one keeps
|
||||
// adding to it, and it is created by the first audio, so a failed start
|
||||
// leaves no file.
|
||||
private recordingName = "";
|
||||
private recorder: WavRecorder | null = null;
|
||||
// The file is open in the controller.
|
||||
private recordingOpen = false;
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false;
|
||||
private readonly recordingListeners = new Set<() => void>();
|
||||
@@ -251,16 +301,55 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* End the continuous capture as an utterance: stop it and have the
|
||||
* transcript of its last audio come as a final. For a capture whose audio
|
||||
* ran out by itself (a replay).
|
||||
*/
|
||||
finishContinuousCapture(): void {
|
||||
this.releaseCapture("continuous", true);
|
||||
}
|
||||
|
||||
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
|
||||
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
|
||||
this.replayListeners.add(listener);
|
||||
return () => this.replayListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Continue the running replay from `positionMs` into its recording. */
|
||||
seekReplay(positionMs: number): void {
|
||||
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. */
|
||||
setReplayPaused(paused: boolean): void {
|
||||
this.controller?.setReplayPaused(paused);
|
||||
}
|
||||
|
||||
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
|
||||
isCaptureLive(): boolean {
|
||||
return this.micIsLive();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or "".
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return;
|
||||
this.recorder?.close();
|
||||
this.recorder = null;
|
||||
this.closeRecording();
|
||||
this.setRecording(name, false);
|
||||
}
|
||||
|
||||
private closeRecording(): void {
|
||||
if (!this.recordingOpen) return;
|
||||
this.recordingOpen = false;
|
||||
try {
|
||||
this.controller?.stopRecording();
|
||||
} catch (error) {
|
||||
console.warn(`Voice: closing the recording failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean {
|
||||
return this.recordingName !== "" && !this.recordingFailure;
|
||||
@@ -286,18 +375,48 @@ export class FaceclawVoiceControlBridge {
|
||||
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||
}
|
||||
|
||||
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||
private record(pcm: any): void {
|
||||
if (!this.recording) return;
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName);
|
||||
if (!this.recorder) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||
/**
|
||||
* Audio arrived for a continuous capture: open the recording asked for, if
|
||||
* it isn't open yet. The controller writes it from then on, as the audio
|
||||
* arrives on its own thread.
|
||||
*/
|
||||
private ensureRecording(): void {
|
||||
if (!this.recording || this.recordingOpen) return;
|
||||
const path = newRecordingPath(this.recordingName);
|
||||
let opened = false;
|
||||
try {
|
||||
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: could not create the recording: ${error}`);
|
||||
}
|
||||
this.recorder.append(pcm);
|
||||
if (!opened) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
this.recordingOpen = true;
|
||||
console.log(`Voice: recording audio to ${path}`);
|
||||
}
|
||||
|
||||
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
|
||||
private syncRecordingEnabled(): void {
|
||||
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
/** The running capture, as the JSON stored in a recording of it. */
|
||||
private describeCapture(): Record<string, unknown> {
|
||||
const options = this.streamOptions;
|
||||
return {
|
||||
recording: this.recordingName,
|
||||
startedAt: new Date().toISOString(),
|
||||
source: options?.usePhoneMic ? "phone" : "glasses",
|
||||
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
|
||||
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
|
||||
processing: {
|
||||
noiseSuppression: Boolean(options?.noiseSuppression),
|
||||
beamFilter: options?.beamFilter ?? null,
|
||||
},
|
||||
...options?.captureInfo,
|
||||
};
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
@@ -364,6 +483,8 @@ export class FaceclawVoiceControlBridge {
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
this.controller?.setMicArray(null);
|
||||
this.controller?.setReplay(null, 0);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
this.controller?.setNoiseSuppression(false);
|
||||
@@ -399,18 +520,33 @@ export class FaceclawVoiceControlBridge {
|
||||
const micLive = this.micIsLive();
|
||||
this.suspendedHolders.delete(holder);
|
||||
this.captureHolders.add(holder);
|
||||
if (micLive && this.streamProvider === transcriptionKey(options)) {
|
||||
if (micLive && this.streamProvider === streamKey(options)) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
this.syncRecordingEnabled();
|
||||
return;
|
||||
}
|
||||
// Any leftover capture is dead, or transcribes with another provider than
|
||||
// Any leftover capture is dead, or captures or transcribes otherwise than
|
||||
// this holder needs; drop it before starting the new one.
|
||||
if (this.started) this.teardownCapture();
|
||||
this.streamProvider = transcriptionKey(options);
|
||||
this.streamProvider = streamKey(options);
|
||||
this.streamOptions = options;
|
||||
this.ensureController();
|
||||
this.syncRecordingEnabled();
|
||||
// A recording under way carries on into this capture: note what changed.
|
||||
if (this.recordingOpen) {
|
||||
try {
|
||||
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: recording event failed: ${error}`);
|
||||
}
|
||||
}
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setMicArray(
|
||||
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
|
||||
);
|
||||
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
@@ -435,7 +571,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.speechPause.reset();
|
||||
// Replaces whatever the last capture left behind ("Listening...", an
|
||||
// error, the permission prompt) until the controller reports progress.
|
||||
this.setStatus("Starting microphone...");
|
||||
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
@@ -480,6 +616,7 @@ export class FaceclawVoiceControlBridge {
|
||||
if (commit) this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
this.syncRecordingEnabled();
|
||||
if (this.captureHolders.size > 0) {
|
||||
// Another holder still wants the mic; keep it running.
|
||||
return;
|
||||
@@ -504,6 +641,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.captureHolders.clear();
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
this.syncRecordingEnabled();
|
||||
this.teardownCapture();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
@@ -521,6 +659,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.suspendedHolders.add(holder);
|
||||
}
|
||||
this.captureHolders.clear();
|
||||
this.syncRecordingEnabled();
|
||||
this.suspendedRaw = this.rawActive;
|
||||
this.teardownCapture();
|
||||
if (this.suspendedHolders.size > 0) {
|
||||
@@ -578,6 +717,7 @@ export class FaceclawVoiceControlBridge {
|
||||
private teardownCapture(): void {
|
||||
this.rawActive = false;
|
||||
this.streamProvider = null;
|
||||
this.streamOptions = null;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
@@ -611,7 +751,7 @@ export class FaceclawVoiceControlBridge {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||
if (this.captureHolders.has("continuous")) this.ensureRecording();
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
@@ -642,6 +782,10 @@ export class FaceclawVoiceControlBridge {
|
||||
);
|
||||
}
|
||||
},
|
||||
onReplay: (kind: string, positionMs: number, durationMs: number) => {
|
||||
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
|
||||
for (const listener of Array.from(this.replayListeners)) listener(event);
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
@@ -678,6 +822,21 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture reads and transcribes with. A running capture is shared only
|
||||
* by holders asking for the same: the provider and language, the microphones
|
||||
* (the mixed stream, or the temples' own under a given configuration), or the
|
||||
* recording it replays.
|
||||
*/
|
||||
function streamKey(options: PushToTalkOptions): string {
|
||||
const source = options.replay
|
||||
? `replay:${options.replay.path}`
|
||||
: options.micArray && !options.usePhoneMic
|
||||
? `array:${Array.from(options.micArray.configure).join(",")}`
|
||||
: "mixed";
|
||||
return `${transcriptionKey(options)}|${source}`;
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
const javaFloats = Array.create("float", values.length);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
@@ -687,3 +846,136 @@ function toJavaFloats(values: Float32Array): any {
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
|
||||
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
|
||||
|
||||
/**
|
||||
* A recording played back as a capture of its own: transcribed by the same
|
||||
* pipeline as the microphone, with the same settings, on a bridge and native
|
||||
* controller separate from the microphone's. Nothing here touches the mic,
|
||||
* so voice input and a live Transcribe window carry on beside a replay.
|
||||
*/
|
||||
export class VoiceReplay {
|
||||
private readonly bridge = new FaceclawVoiceControlBridge();
|
||||
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
|
||||
private readonly offReplay: () => void;
|
||||
private position = 0;
|
||||
private duration = 0;
|
||||
private current: VoiceReplayState = "opening";
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* `options` gives the capture options (provider, keys, processing) each
|
||||
* time the replay starts, so a restart picks up changed settings.
|
||||
*/
|
||||
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
|
||||
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
|
||||
}
|
||||
|
||||
/** The replay's status, transcript and level events. */
|
||||
get events(): VoiceEventSource {
|
||||
return this.bridge;
|
||||
}
|
||||
|
||||
/** Milliseconds into the recording. */
|
||||
get positionMs(): number {
|
||||
return this.position;
|
||||
}
|
||||
|
||||
/** The recording's length in ms; 0 until it has been opened. */
|
||||
get durationMs(): number {
|
||||
return this.duration;
|
||||
}
|
||||
|
||||
get state(): VoiceReplayState {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/**
|
||||
* The position, duration or state changed. `jumpedToMs` is given when
|
||||
* playback moved there other than by playing on: a jump, or a start
|
||||
* part-way in. It comes after the transcript of what played before.
|
||||
*/
|
||||
onChange(listener: (jumpedToMs?: number) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Play from `positionMs` into the recording. */
|
||||
start(positionMs = 0): void {
|
||||
if (this.closed) return;
|
||||
this.position = Math.max(0, positionMs);
|
||||
this.set("opening");
|
||||
this.bridge.startContinuousCapture({
|
||||
...this.options(),
|
||||
communicator: null,
|
||||
usePhoneMic: false,
|
||||
micArray: null,
|
||||
replay: { path: this.path, startMs: this.position },
|
||||
});
|
||||
}
|
||||
|
||||
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
|
||||
jumpTo(positionMs: number): void {
|
||||
if (this.closed) return;
|
||||
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
|
||||
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
|
||||
this.bridge.seekReplay(target);
|
||||
this.position = target;
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.start(target);
|
||||
}
|
||||
|
||||
setPaused(paused: boolean): void {
|
||||
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
|
||||
this.bridge.setReplayPaused(paused);
|
||||
this.set(paused ? "paused" : "playing");
|
||||
}
|
||||
|
||||
/** Stop for good and release the capture. */
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.offReplay();
|
||||
this.listeners.clear();
|
||||
this.bridge.stop();
|
||||
}
|
||||
|
||||
private handle(event: VoiceReplayEvent): void {
|
||||
this.duration = event.durationMs;
|
||||
switch (event.kind) {
|
||||
case "start":
|
||||
this.position = event.positionMs;
|
||||
this.current = "playing";
|
||||
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
|
||||
return;
|
||||
case "seek":
|
||||
this.position = event.positionMs;
|
||||
this.notify(event.positionMs);
|
||||
return;
|
||||
case "end":
|
||||
this.position = event.durationMs;
|
||||
// Its last audio is still to be transcribed (a cloud provider's final).
|
||||
this.bridge.finishContinuousCapture();
|
||||
this.set("ended");
|
||||
return;
|
||||
case "error":
|
||||
this.set("failed");
|
||||
return;
|
||||
default:
|
||||
this.position = event.positionMs;
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private set(state: VoiceReplayState): void {
|
||||
this.current = state;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
private notify(jumpedToMs?: number): void {
|
||||
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
|
||||
export function wavHeader(dataBytes: number): Uint8Array {
|
||||
const header = new Uint8Array(44);
|
||||
const view = new DataView(header.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataBytes, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, 16000, true);
|
||||
view.setUint32(28, 32000, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, dataBytes, true);
|
||||
return header;
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
import { wavHeader } from './wav-header'
|
||||
|
||||
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
|
||||
const FLUSH_BYTES = 32000
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
|
||||
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private pending: Uint8Array[] = []
|
||||
private pendingBytes = 0
|
||||
private dataBytes = 0
|
||||
|
||||
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.wav`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
|
||||
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
|
||||
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
|
||||
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
|
||||
return new WavRecorder(handle, path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.handle || !pcm.length) return
|
||||
this.pending.push(pcm.slice())
|
||||
this.pendingBytes += pcm.length
|
||||
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.flush()
|
||||
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
|
||||
this.handle = null
|
||||
}
|
||||
|
||||
private flush(): void {
|
||||
const handle = this.handle
|
||||
if (!handle || !this.pendingBytes) return
|
||||
const chunk = new Uint8Array(this.pendingBytes)
|
||||
let offset = 0
|
||||
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
|
||||
this.pending = []
|
||||
this.pendingBytes = 0
|
||||
try {
|
||||
// NativeScript throws a failed call's NSError (storage full, file removed).
|
||||
handle.writeDataError(toData(chunk))
|
||||
handle.seekToOffsetError(0)
|
||||
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
|
||||
handle.seekToEndReturningOffsetError(null)
|
||||
this.dataBytes += chunk.length
|
||||
return
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
}
|
||||
try { handle.closeAndReturnError() } catch { /* ignore */ }
|
||||
this.handle = null
|
||||
}
|
||||
}
|
||||
|
||||
function toData(bytes: Uint8Array): NSData {
|
||||
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
declare const com: any;
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}.wav`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||
const path = String(file.getAbsolutePath());
|
||||
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||
} catch (error) {
|
||||
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||
append(pcm: any): void {
|
||||
this.recorder?.append(pcm);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.recorder?.finish();
|
||||
this.recorder = null;
|
||||
}
|
||||
}
|
||||
@@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
|
||||
storageKey: "developer.useMicControl",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
|
||||
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
|
||||
});
|
||||
|
||||
export const showBleBandwidthSetting = new ConfigSettingBoolean({
|
||||
@@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
|
||||
label: "Save Recording",
|
||||
storageKey: "voice.saveRecording",
|
||||
defaultValue: false,
|
||||
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
|
||||
description: global.isIOS
|
||||
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
|
||||
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
|
||||
+19
-1
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
|
||||
import { dimPlanes, type Plane } from "../graphics/plane";
|
||||
import { spanCurrent } from "../native/frame-timings";
|
||||
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
|
||||
import { type VoiceReplay } from "../native/voice-control";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
|
||||
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
|
||||
|
||||
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
|
||||
|
||||
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
||||
export type ContinuousCaptureOptions = {
|
||||
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
|
||||
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
|
||||
recordAs?: string;
|
||||
/**
|
||||
* How many of the glasses' microphones to capture: 1 is their own mixed
|
||||
* stream; 2 (one per temple) and 4 (both of each temple) stream the
|
||||
* microphones separately through the custom firmware, for recording and
|
||||
* array processing. While any client asks for more than one, every capture
|
||||
* uses the most asked for. Glasses without the microphone channel, and the
|
||||
* phone's microphone, give 1 whatever is asked.
|
||||
*/
|
||||
mics?: MicCount;
|
||||
/**
|
||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||
* or "" for the provider to detect it), instead of the Transcription
|
||||
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
|
||||
|
||||
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
|
||||
|
||||
export type MicCount = 1 | 2 | 4;
|
||||
|
||||
export type LayerActions = {
|
||||
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
|
||||
requestRender: () => void;
|
||||
@@ -55,6 +67,12 @@ export type LayerActions = {
|
||||
*/
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
/**
|
||||
* Open a recording (.mka) for playing back through transcription in place
|
||||
* of the microphone; the caller starts and closes it. Absent, or null,
|
||||
* where recordings can't be replayed.
|
||||
*/
|
||||
openVoiceReplay?: (path: string) => VoiceReplay | null;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
|
||||
import { truncateText } from "../graphics/textwrap";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
|
||||
import { lineStep } from "./metrics";
|
||||
import type { Edges } from "./viewport";
|
||||
|
||||
@@ -68,8 +68,9 @@ export class MicLevelMeter {
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
|
||||
/** `source` is the capture whose level to show: the microphone's unless given. */
|
||||
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
|
||||
this.unsubscribe = source.onMicLevel((level) => {
|
||||
this.peak = Math.max(this.peak, level);
|
||||
});
|
||||
this.timer = setInterval(() => {
|
||||
|
||||
@@ -48,6 +48,8 @@ export type InProcessWindowOptions = {
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** See ShellWindow.keepsScreenOn. */
|
||||
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||
/** See ShellWindow.restoresApp. */
|
||||
restoresApp?: boolean;
|
||||
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||
onForegroundChanged?: (foreground: boolean) => void;
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
@@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
keepsScreenOn: options.keepsScreenOn,
|
||||
restoresApp: options.restoresApp,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
|
||||
@@ -151,6 +151,13 @@ export type ShellWindow = {
|
||||
* draws this window's top bar.
|
||||
*/
|
||||
heightMode: WindowHeightMode;
|
||||
/**
|
||||
* False for a window that must not bring its app back when Faceclaw
|
||||
* restarts (the controller's open-apps restore relaunches apps, not
|
||||
* windows): one that shows something a plain launch of its app would not,
|
||||
* where the relaunch would do something else in its place.
|
||||
*/
|
||||
restoresApp?: boolean;
|
||||
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
|
||||
close?: () => void;
|
||||
drawIcon: ShellChromeWindow["drawIcon"];
|
||||
|
||||
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
|
||||
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
|
||||
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
|
||||
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
|
||||
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
|
||||
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
|
||||
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
|
||||
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
|
||||
|
||||
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
|
||||
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
|
||||
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
file.write(bytes, offset, length)
|
||||
size += length
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
file.seek(position)
|
||||
file.write(bytes)
|
||||
file.seek(size)
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
|
||||
private val file = java.io.RandomAccessFile(path, "r")
|
||||
override val length = file.length()
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
file.seek(position)
|
||||
var total = 0
|
||||
while (total < length) {
|
||||
val read = file.read(buffer, offset + total, length - total)
|
||||
if (read <= 0) break
|
||||
total += read
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
|
||||
expect fun deleteFile(path: String): Boolean
|
||||
|
||||
expect fun renameFile(from: String, to: String): Boolean
|
||||
|
||||
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
|
||||
interface SeekableOutput {
|
||||
/** Bytes appended so far. */
|
||||
val size: Long
|
||||
|
||||
/** Appends [length] bytes of [bytes] from [offset]. */
|
||||
fun write(bytes: ByteArray, offset: Int, length: Int)
|
||||
|
||||
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
|
||||
fun writeAt(position: Long, bytes: ByteArray)
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** A file read at arbitrary positions without loading it whole. */
|
||||
interface SeekableInput {
|
||||
val length: Long
|
||||
|
||||
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
|
||||
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
|
||||
expect fun openFileOutput(path: String): SeekableOutput
|
||||
|
||||
/** Opens [path] for reading. Throws when it cannot. */
|
||||
expect fun openFileInput(path: String): SeekableInput
|
||||
@@ -0,0 +1,764 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/**
|
||||
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
|
||||
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
|
||||
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
|
||||
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
|
||||
*
|
||||
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
|
||||
* about a second each, cues and the segment size written when the recording finishes. A
|
||||
* recording cut short by a crash is still a valid live-style stream of everything up to its last
|
||||
* complete cluster, and the reader takes it as such.
|
||||
*/
|
||||
internal object MatroskaIds {
|
||||
const val EBML = 0x1A45DFA3
|
||||
const val EBML_VERSION = 0x4286
|
||||
const val EBML_READ_VERSION = 0x42F7
|
||||
const val EBML_MAX_ID_LENGTH = 0x42F2
|
||||
const val EBML_MAX_SIZE_LENGTH = 0x42F3
|
||||
const val DOC_TYPE = 0x4282
|
||||
const val DOC_TYPE_VERSION = 0x4287
|
||||
const val DOC_TYPE_READ_VERSION = 0x4285
|
||||
const val VOID = 0xEC
|
||||
const val SEGMENT = 0x18538067
|
||||
const val SEEK_HEAD = 0x114D9B74
|
||||
const val SEEK = 0x4DBB
|
||||
const val SEEK_ID = 0x53AB
|
||||
const val SEEK_POSITION = 0x53AC
|
||||
const val INFO = 0x1549A966
|
||||
const val TIMESTAMP_SCALE = 0x2AD7B1
|
||||
const val DURATION = 0x4489
|
||||
const val DATE_UTC = 0x4461
|
||||
const val TITLE = 0x7BA9
|
||||
const val MUXING_APP = 0x4D80
|
||||
const val WRITING_APP = 0x5741
|
||||
const val TRACKS = 0x1654AE6B
|
||||
const val TRACK_ENTRY = 0xAE
|
||||
const val TRACK_NUMBER = 0xD7
|
||||
const val TRACK_UID = 0x73C5
|
||||
const val TRACK_TYPE = 0x83
|
||||
const val FLAG_DEFAULT = 0x88
|
||||
const val FLAG_LACING = 0x9C
|
||||
const val NAME = 0x536E
|
||||
const val LANGUAGE = 0x22B59C
|
||||
const val CODEC_ID = 0x86
|
||||
const val AUDIO = 0xE1
|
||||
const val SAMPLING_FREQUENCY = 0xB5
|
||||
const val CHANNELS = 0x9F
|
||||
const val BIT_DEPTH = 0x6264
|
||||
const val ATTACHMENTS = 0x1941A469
|
||||
const val ATTACHED_FILE = 0x61A7
|
||||
const val FILE_NAME = 0x466E
|
||||
const val FILE_MIME_TYPE = 0x4660
|
||||
const val FILE_DATA = 0x465C
|
||||
const val FILE_UID = 0x46AE
|
||||
const val CLUSTER = 0x1F43B675
|
||||
const val TIMESTAMP = 0xE7
|
||||
const val SIMPLE_BLOCK = 0xA3
|
||||
const val BLOCK_GROUP = 0xA0
|
||||
const val BLOCK = 0xA1
|
||||
const val BLOCK_DURATION = 0x9B
|
||||
const val CUES = 0x1C53BB6B
|
||||
const val CUE_POINT = 0xBB
|
||||
const val CUE_TIME = 0xB3
|
||||
const val CUE_TRACK_POSITIONS = 0xB7
|
||||
const val CUE_TRACK = 0xF7
|
||||
const val CUE_CLUSTER_POSITION = 0xF1
|
||||
}
|
||||
|
||||
/** Builds EBML elements into a byte buffer. */
|
||||
internal class EbmlWriter(capacity: Int = 256) {
|
||||
private val sink = ByteSink(capacity)
|
||||
|
||||
fun size(): Int = sink.size()
|
||||
|
||||
fun bytes(): ByteArray = sink.toByteArray()
|
||||
|
||||
/** An element ID, written as given: its length marker is part of the value. */
|
||||
fun id(id: Int) {
|
||||
if (id > 0xFFFFFF) sink.write(id ushr 24)
|
||||
if (id > 0xFFFF) sink.write(id ushr 16)
|
||||
if (id > 0xFF) sink.write(id ushr 8)
|
||||
sink.write(id)
|
||||
}
|
||||
|
||||
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
|
||||
fun dataSize(size: Long, width: Int = 0) {
|
||||
var bytes = width
|
||||
if (bytes == 0) {
|
||||
bytes = 1
|
||||
// The all-ones value of each width means "unknown size".
|
||||
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
|
||||
}
|
||||
val marked = size or (1L shl (7 * bytes))
|
||||
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun uint(id: Int, value: Long) {
|
||||
var bytes = 1
|
||||
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
|
||||
id(id)
|
||||
dataSize(bytes.toLong())
|
||||
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
/** A fixed eight-byte integer (dates). */
|
||||
fun int64(id: Int, value: Long) {
|
||||
id(id)
|
||||
dataSize(8)
|
||||
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
|
||||
|
||||
fun string(id: Int, text: String) {
|
||||
val bytes = text.encodeToByteArray()
|
||||
binary(id, bytes, 0, bytes.size)
|
||||
}
|
||||
|
||||
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
|
||||
id(id)
|
||||
dataSize(length.toLong())
|
||||
sink.write(bytes, offset, length)
|
||||
}
|
||||
|
||||
fun raw(bytes: ByteArray) = sink.write(bytes)
|
||||
|
||||
fun raw(value: Int) = sink.write(value)
|
||||
|
||||
fun master(id: Int, build: EbmlWriter.() -> Unit) {
|
||||
val child = EbmlWriter()
|
||||
child.build()
|
||||
id(id)
|
||||
dataSize(child.size().toLong())
|
||||
sink.write(child.bytes())
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
|
||||
fun padded(element: ByteArray, reserved: Int): ByteArray? {
|
||||
val filler = reserved - element.size
|
||||
// A Void element is at least two bytes.
|
||||
if (filler < 0 || filler == 1) return null
|
||||
val out = element.copyOf(reserved)
|
||||
if (filler == 0) return out
|
||||
val at = element.size
|
||||
out[at] = MatroskaIds.VOID.toByte()
|
||||
if (filler < 9) {
|
||||
out[at + 1] = (0x80 or (filler - 2)).toByte()
|
||||
} else {
|
||||
val size = (filler - 9).toLong()
|
||||
out[at + 1] = 0x01
|
||||
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
|
||||
|
||||
/**
|
||||
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
|
||||
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
|
||||
* milliseconds from the start of the file. Not thread-safe.
|
||||
*/
|
||||
class MkaWriter(
|
||||
private val output: SeekableOutput,
|
||||
title: String?,
|
||||
/** When the recording began (Unix epoch ms), for the file's date. */
|
||||
dateUtcMs: Long,
|
||||
attachments: List<MkaAttachment>,
|
||||
) {
|
||||
companion object {
|
||||
const val CODEC_PCM = "A_PCM/INT/LIT"
|
||||
const val CODEC_TEXT = "S_TEXT/UTF8"
|
||||
const val TRACK_AUDIO = 2
|
||||
const val TRACK_SUBTITLE = 0x11
|
||||
const val APP_NAME = "Faceclaw"
|
||||
private const val SEEK_RESERVED = 128
|
||||
private const val TRACKS_RESERVED = 4096
|
||||
private const val CLUSTER_SPAN_MS = 1000L
|
||||
private const val CLUSTER_MAX_BYTES = 1 shl 20
|
||||
// Block timestamps are signed 16-bit offsets from their cluster's.
|
||||
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
|
||||
private const val DURATION_PATCH_CLUSTERS = 5
|
||||
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
|
||||
private const val DATE_EPOCH_MS = 978_307_200_000L
|
||||
}
|
||||
|
||||
private class Track(
|
||||
val number: Int,
|
||||
val type: Int,
|
||||
val name: String,
|
||||
val sampleRate: Int,
|
||||
val channels: Int,
|
||||
val preferred: Boolean,
|
||||
)
|
||||
|
||||
private class Cue(val clusterPosition: Long, val timeMs: Long)
|
||||
|
||||
private val tracks = ArrayList<Track>()
|
||||
private val cues = ArrayList<Cue>()
|
||||
private val segmentSizePosition: Long
|
||||
private val segmentStart: Long
|
||||
private val seekPosition: Long
|
||||
private val infoPosition: Long
|
||||
private val durationPosition: Long
|
||||
private val tracksPosition: Long
|
||||
private val attachmentsPosition: Long
|
||||
private var cluster: EbmlWriter? = null
|
||||
private var clusterBaseMs = 0L
|
||||
private var clustersSincePatch = 0
|
||||
private var finished = false
|
||||
private val hasAttachments = attachments.isNotEmpty()
|
||||
|
||||
/** The end of the latest audio or text written, in ms from the start of the file. */
|
||||
var durationMs = 0L
|
||||
private set
|
||||
|
||||
init {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.EBML) {
|
||||
uint(MatroskaIds.EBML_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_READ_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
|
||||
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
|
||||
string(MatroskaIds.DOC_TYPE, "matroska")
|
||||
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
|
||||
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
|
||||
}
|
||||
head.id(MatroskaIds.SEGMENT)
|
||||
segmentSizePosition = head.size().toLong()
|
||||
// Unknown size (a live stream) until finish() knows it.
|
||||
head.raw(0x01)
|
||||
repeat(7) { head.raw(0xFF) }
|
||||
segmentStart = head.size().toLong()
|
||||
append(head.bytes())
|
||||
|
||||
seekPosition = output.size
|
||||
infoPosition = seekPosition + SEEK_RESERVED
|
||||
val info = EbmlWriter()
|
||||
info.master(MatroskaIds.INFO) {
|
||||
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
|
||||
string(MatroskaIds.MUXING_APP, APP_NAME)
|
||||
string(MatroskaIds.WRITING_APP, APP_NAME)
|
||||
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
|
||||
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
|
||||
// Last, so its eight bytes are the element's final ones.
|
||||
float64(MatroskaIds.DURATION, 0.0)
|
||||
}
|
||||
durationPosition = infoPosition + info.size() - 8
|
||||
tracksPosition = infoPosition + info.size()
|
||||
attachmentsPosition = tracksPosition + TRACKS_RESERVED
|
||||
|
||||
append(seekHead(-1))
|
||||
append(info.bytes())
|
||||
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
|
||||
if (attachments.isNotEmpty()) {
|
||||
val files = EbmlWriter()
|
||||
files.master(MatroskaIds.ATTACHMENTS) {
|
||||
attachments.forEachIndexed { index, file ->
|
||||
master(MatroskaIds.ATTACHED_FILE) {
|
||||
string(MatroskaIds.FILE_NAME, file.name)
|
||||
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
|
||||
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
|
||||
uint(MatroskaIds.FILE_UID, index + 1L)
|
||||
}
|
||||
}
|
||||
}
|
||||
append(files.bytes())
|
||||
}
|
||||
}
|
||||
|
||||
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
|
||||
|
||||
private fun seekHead(cuesPosition: Long): ByteArray {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.SEEK_HEAD) {
|
||||
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
|
||||
val idBytes = EbmlWriter()
|
||||
idBytes.id(id)
|
||||
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
|
||||
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
|
||||
}
|
||||
entry(MatroskaIds.INFO, infoPosition)
|
||||
entry(MatroskaIds.TRACKS, tracksPosition)
|
||||
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
|
||||
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
|
||||
}
|
||||
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
|
||||
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
|
||||
* the one players pick by default.
|
||||
*/
|
||||
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
|
||||
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
|
||||
|
||||
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
|
||||
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
|
||||
|
||||
private fun addTrack(track: Track): Int {
|
||||
check(!finished) { "recording already finished" }
|
||||
// A block's track number is written in one byte.
|
||||
if (track.number > 126) return -1
|
||||
tracks.add(track)
|
||||
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
|
||||
if (region == null) {
|
||||
tracks.remove(track)
|
||||
return -1
|
||||
}
|
||||
output.writeAt(tracksPosition, region)
|
||||
return track.number
|
||||
}
|
||||
|
||||
private fun listedTracks(): List<Track> =
|
||||
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
|
||||
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
|
||||
tracks.filter { it.type != TRACK_AUDIO }
|
||||
|
||||
private fun tracksElement(): ByteArray {
|
||||
val listed = listedTracks()
|
||||
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
|
||||
val out = EbmlWriter(1024)
|
||||
out.master(MatroskaIds.TRACKS) {
|
||||
for (track in listed) {
|
||||
master(MatroskaIds.TRACK_ENTRY) {
|
||||
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_UID, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
|
||||
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
|
||||
uint(MatroskaIds.FLAG_LACING, 0)
|
||||
string(MatroskaIds.NAME, track.name)
|
||||
string(MatroskaIds.LANGUAGE, "und")
|
||||
if (track.type == TRACK_AUDIO) {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_PCM)
|
||||
master(MatroskaIds.AUDIO) {
|
||||
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
|
||||
uint(MatroskaIds.CHANNELS, track.channels.toLong())
|
||||
uint(MatroskaIds.BIT_DEPTH, 16)
|
||||
}
|
||||
} else {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.bytes()
|
||||
}
|
||||
|
||||
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
|
||||
|
||||
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
|
||||
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
|
||||
val track = track(trackNumber)
|
||||
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
|
||||
if (length <= 0) return
|
||||
val block = openCluster(timeMs)
|
||||
block.id(MatroskaIds.SIMPLE_BLOCK)
|
||||
block.dataSize(4L + length)
|
||||
blockHeader(block, trackNumber, timeMs, 0x80)
|
||||
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
|
||||
val frames = length / (2 * track.channels)
|
||||
noteEnd(timeMs + frames * 1000L / track.sampleRate)
|
||||
}
|
||||
|
||||
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
|
||||
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
|
||||
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
|
||||
val bytes = text.encodeToByteArray()
|
||||
val block = openCluster(timeMs)
|
||||
block.master(MatroskaIds.BLOCK_GROUP) {
|
||||
id(MatroskaIds.BLOCK)
|
||||
dataSize(4L + bytes.size)
|
||||
blockHeader(this, trackNumber, timeMs, 0)
|
||||
raw(bytes)
|
||||
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
|
||||
}
|
||||
noteEnd(timeMs + maxOf(1L, durationMs))
|
||||
}
|
||||
|
||||
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
|
||||
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
|
||||
block.raw(0x80 or trackNumber)
|
||||
block.raw(relative shr 8)
|
||||
block.raw(relative)
|
||||
block.raw(flags)
|
||||
}
|
||||
|
||||
private fun noteEnd(endMs: Long) {
|
||||
if (endMs > durationMs) durationMs = endMs
|
||||
}
|
||||
|
||||
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
|
||||
private fun openCluster(timeMs: Long): EbmlWriter {
|
||||
check(!finished) { "recording already finished" }
|
||||
val time = maxOf(0L, timeMs)
|
||||
val open = cluster
|
||||
if (open != null) {
|
||||
val relative = time - clusterBaseMs
|
||||
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
|
||||
flushCluster()
|
||||
}
|
||||
clusterBaseMs = time
|
||||
return EbmlWriter(64 * 1024).also { cluster = it }
|
||||
}
|
||||
|
||||
private fun flushCluster() {
|
||||
val body = cluster ?: return
|
||||
cluster = null
|
||||
if (body.size() == 0) return
|
||||
val timestamp = EbmlWriter(16)
|
||||
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
|
||||
val header = EbmlWriter(32)
|
||||
header.id(MatroskaIds.CLUSTER)
|
||||
header.dataSize(timestamp.size().toLong() + body.size())
|
||||
header.raw(timestamp.bytes())
|
||||
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
|
||||
append(header.bytes())
|
||||
append(body.bytes())
|
||||
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
|
||||
}
|
||||
|
||||
private fun patchDuration() {
|
||||
clustersSincePatch = 0
|
||||
val value = durationMs.toDouble().toRawBits()
|
||||
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
|
||||
}
|
||||
|
||||
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
|
||||
fun finish() {
|
||||
if (finished) return
|
||||
try {
|
||||
flushCluster()
|
||||
finished = true
|
||||
val cuesPosition = output.size
|
||||
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
|
||||
if (cues.isNotEmpty()) {
|
||||
val index = EbmlWriter(cues.size * 16 + 16)
|
||||
index.master(MatroskaIds.CUES) {
|
||||
for (cue in cues) {
|
||||
master(MatroskaIds.CUE_POINT) {
|
||||
uint(MatroskaIds.CUE_TIME, cue.timeMs)
|
||||
master(MatroskaIds.CUE_TRACK_POSITIONS) {
|
||||
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
|
||||
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
append(index.bytes())
|
||||
output.writeAt(seekPosition, seekHead(cuesPosition))
|
||||
}
|
||||
patchDuration()
|
||||
val size = EbmlWriter(8)
|
||||
size.dataSize(output.size - segmentStart, 8)
|
||||
output.writeAt(segmentSizePosition, size.bytes())
|
||||
} finally {
|
||||
finished = true
|
||||
output.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaTrackInfo(
|
||||
@JvmField val number: Int,
|
||||
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
|
||||
@JvmField val type: Int,
|
||||
@JvmField val codecId: String,
|
||||
@JvmField val name: String,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val bitDepth: Int,
|
||||
) {
|
||||
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
|
||||
}
|
||||
|
||||
class MkaBlock(
|
||||
@JvmField val track: Int,
|
||||
@JvmField val timeMs: Long,
|
||||
@JvmField val data: ByteArray,
|
||||
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
|
||||
@JvmField val durationMs: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
|
||||
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
|
||||
* IllegalArgumentException for anything that is not such a file.
|
||||
*/
|
||||
class MkaReader(private val input: SeekableInput) {
|
||||
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
|
||||
|
||||
private val clusters = ArrayList<ClusterRef>()
|
||||
private val trackList = ArrayList<MkaTrackInfo>()
|
||||
private val attachmentList = ArrayList<MkaAttachment>()
|
||||
private var timestampScale = 1_000_000L
|
||||
private var infoDurationMs = 0L
|
||||
private var nextCluster = 0
|
||||
private var blocks: List<MkaBlock> = emptyList()
|
||||
private var blockIndex = 0
|
||||
private var skipBeforeMs = 0L
|
||||
|
||||
val tracks: List<MkaTrackInfo> get() = trackList
|
||||
val attachments: List<MkaAttachment> get() = attachmentList
|
||||
|
||||
/** The end of the last audio or text in the file, in ms. */
|
||||
val durationMs: Long
|
||||
|
||||
init {
|
||||
scan()
|
||||
var end = infoDurationMs
|
||||
if (clusters.isNotEmpty()) {
|
||||
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
|
||||
}
|
||||
durationMs = end
|
||||
}
|
||||
|
||||
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
|
||||
|
||||
/** Continue from the first block at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
|
||||
nextCluster = maxOf(0, index)
|
||||
blocks = emptyList()
|
||||
blockIndex = 0
|
||||
skipBeforeMs = timeMs
|
||||
}
|
||||
|
||||
/** The next block, or null at the end of the file. */
|
||||
fun next(): MkaBlock? {
|
||||
while (true) {
|
||||
while (blockIndex >= blocks.size) {
|
||||
if (nextCluster >= clusters.size) return null
|
||||
blocks = readCluster(clusters[nextCluster++])
|
||||
blockIndex = 0
|
||||
}
|
||||
val block = blocks[blockIndex++]
|
||||
if (block.timeMs >= skipBeforeMs) return block
|
||||
}
|
||||
}
|
||||
|
||||
fun close() = input.close()
|
||||
|
||||
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
|
||||
|
||||
private fun blockSpanMs(block: MkaBlock): Long {
|
||||
if (block.durationMs >= 0) return block.durationMs
|
||||
val track = track(block.track) ?: return 0
|
||||
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
|
||||
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
|
||||
}
|
||||
|
||||
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
|
||||
private class Header(val id: Int, val size: Long, val length: Int)
|
||||
|
||||
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
|
||||
if (at >= end) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val idLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (idLength > 4 || at + idLength >= end) return null
|
||||
var id = 0
|
||||
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
|
||||
val sizeAt = at + idLength
|
||||
val sizeFirst = bytes[sizeAt].toInt() and 0xff
|
||||
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
|
||||
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
|
||||
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
|
||||
var allOnes = size == (0xff shr sizeLength).toLong()
|
||||
for (index in 1 until sizeLength) {
|
||||
val byte = bytes[sizeAt + index].toInt() and 0xff
|
||||
if (byte != 0xff) allOnes = false
|
||||
size = (size shl 8) or byte.toLong()
|
||||
}
|
||||
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
|
||||
}
|
||||
|
||||
private fun readAt(position: Long, length: Int): ByteArray {
|
||||
val buffer = ByteArray(length)
|
||||
val read = input.readAt(position, buffer, 0, length)
|
||||
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
|
||||
}
|
||||
|
||||
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
|
||||
var value = 0L
|
||||
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
|
||||
return value
|
||||
}
|
||||
|
||||
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
|
||||
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
|
||||
8 -> Double.fromBits(uint(bytes, at, 8))
|
||||
else -> 0.0
|
||||
}
|
||||
|
||||
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
|
||||
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
|
||||
var at = start
|
||||
while (true) {
|
||||
val child = header(bytes, at, end) ?: return
|
||||
val payload = at + child.length
|
||||
if (child.size < 0 || child.size > end - payload) return
|
||||
visit(child.id, payload, child.size.toInt())
|
||||
at = payload + child.size.toInt()
|
||||
}
|
||||
}
|
||||
|
||||
private fun scan() {
|
||||
val length = input.length
|
||||
val start = readAt(0, 64)
|
||||
val ebml = header(start, 0, start.size)
|
||||
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
|
||||
var position = ebml.length + ebml.size
|
||||
val segment = readAt(position, 12).let { header(it, 0, it.size) }
|
||||
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
|
||||
position += segment.length
|
||||
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
|
||||
while (position < end) {
|
||||
val head = readAt(position, 12)
|
||||
val element = header(head, 0, head.size) ?: break
|
||||
val payload = position + element.length
|
||||
if (element.id == MatroskaIds.CLUSTER) {
|
||||
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
|
||||
// A recording cut short ends in a partial cluster; its complete blocks still count.
|
||||
val available = minOf(element.size, end - payload)
|
||||
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
|
||||
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
|
||||
} else if (element.size < 0) {
|
||||
break
|
||||
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
|
||||
when (element.id) {
|
||||
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
|
||||
}
|
||||
}
|
||||
if (element.size < 0) break
|
||||
position = payload + element.size
|
||||
}
|
||||
}
|
||||
|
||||
private fun clusterTime(position: Long, size: Int): Long {
|
||||
val head = readAt(position, minOf(size, 16))
|
||||
val first = header(head, 0, head.size)
|
||||
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
|
||||
return uint(head, first.length, first.size.toInt())
|
||||
}
|
||||
val whole = readAt(position, size)
|
||||
var time = 0L
|
||||
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
|
||||
return time
|
||||
}
|
||||
|
||||
private fun parseInfo(bytes: ByteArray) {
|
||||
var duration = 0.0
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
|
||||
MatroskaIds.DURATION -> duration = float(bytes, at, size)
|
||||
}
|
||||
}
|
||||
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
|
||||
}
|
||||
|
||||
private fun parseTracks(bytes: ByteArray) {
|
||||
trackList.clear()
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.TRACK_ENTRY) {
|
||||
var number = 0
|
||||
var type = 0
|
||||
var codec = ""
|
||||
var name = ""
|
||||
var rate = 8000.0
|
||||
var channels = 1
|
||||
var depth = 0
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
|
||||
when (audio) {
|
||||
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
|
||||
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
|
||||
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val audio = type == MkaWriter.TRACK_AUDIO
|
||||
trackList.add(
|
||||
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseAttachments(bytes: ByteArray) {
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.ATTACHED_FILE) {
|
||||
var name = ""
|
||||
var mime = ""
|
||||
var data = ByteArray(0)
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
|
||||
}
|
||||
}
|
||||
attachmentList.add(MkaAttachment(name, mime, data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
|
||||
val bytes = readAt(cluster.position, cluster.size)
|
||||
val out = ArrayList<MkaBlock>()
|
||||
var clusterRaw = cluster.rawTime
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
|
||||
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
|
||||
MatroskaIds.BLOCK_GROUP -> {
|
||||
var blockAt = -1
|
||||
var blockSize = 0
|
||||
var duration = -1L
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.BLOCK -> {
|
||||
blockAt = fieldAt
|
||||
blockSize = fieldSize
|
||||
}
|
||||
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
|
||||
}
|
||||
}
|
||||
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
|
||||
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
|
||||
if (size < 4) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val trackLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (trackLength > 2 || size < trackLength + 3) return null
|
||||
var track = first and (0xff shr trackLength)
|
||||
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
|
||||
val head = at + trackLength
|
||||
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
|
||||
val flags = bytes[head + 2].toInt() and 0xff
|
||||
if (flags and 0x06 != 0) return null
|
||||
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
|
||||
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
|
||||
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
|
||||
* contract is the header of g2flash patches/mic_control.c).
|
||||
*/
|
||||
object MicArrayProtocol {
|
||||
const val VERSION = 1
|
||||
const val HEADER_BYTES = 21
|
||||
const val OP_CONFIGURE = 1
|
||||
const val OP_STOP = 3
|
||||
const val OP_RENEW = 4
|
||||
/** How long an armed temple streams without a CONFIGURE or RENEW. */
|
||||
const val LEASE_MS = 90_000L
|
||||
const val FLAG_TRUNCATED = 0x80
|
||||
const val CODEC_RAW = 1
|
||||
const val FORMAT_PCM16 = 0
|
||||
|
||||
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
|
||||
class Frame(
|
||||
/** Low 16 bits of the temple's frame counter. */
|
||||
@JvmField val sequence: Int,
|
||||
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
|
||||
@JvmField val deviceTickMs: Long,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val format: Int,
|
||||
@JvmField val codec: Int,
|
||||
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
|
||||
@JvmField val angleDegrees: Int,
|
||||
@JvmField val ssr: Int,
|
||||
/** The firmware cut the payload short of the samples it captured. */
|
||||
@JvmField val truncated: Boolean,
|
||||
@JvmField val payloadOffset: Int,
|
||||
@JvmField val payloadLength: Int,
|
||||
)
|
||||
|
||||
/** A control record with no payload: ['M','C', version, op]. */
|
||||
@JvmStatic
|
||||
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
|
||||
|
||||
@JvmStatic
|
||||
fun isConfigure(record: ByteArray?): Boolean =
|
||||
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
|
||||
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
|
||||
|
||||
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
|
||||
@JvmStatic
|
||||
fun decodeFrame(data: ByteArray): Frame? {
|
||||
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
|
||||
return null
|
||||
}
|
||||
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
|
||||
return Frame(
|
||||
sequence = u16(data, 4),
|
||||
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
|
||||
channels = data[10].toInt() and 0xff,
|
||||
sampleRate = u16(data, 11) * 100,
|
||||
format = data[13].toInt() and 0xff,
|
||||
codec = data[14].toInt() and 0xff,
|
||||
angleDegrees = u16(data, 15).toShort().toInt(),
|
||||
ssr = u16(data, 17).toShort().toInt(),
|
||||
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
|
||||
payloadOffset = HEADER_BYTES,
|
||||
payloadLength = length,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
|
||||
* payload isn't raw 16-bit PCM or holds none.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
|
||||
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
|
||||
val frames = frame.payloadLength / (2 * frame.channels)
|
||||
if (frames <= 0) return null
|
||||
val samples = ShortArray(frames * frame.channels)
|
||||
var at = frame.payloadOffset
|
||||
for (index in samples.indices) {
|
||||
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
|
||||
at += 2
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.concurrent.Volatile
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.abs
|
||||
|
||||
/** Which microphones a chunk of capture audio came from. */
|
||||
object MicStream {
|
||||
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
|
||||
const val MONO = 0
|
||||
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
|
||||
const val LEFT = 1
|
||||
const val RIGHT = 2
|
||||
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
|
||||
const val MIX = 3
|
||||
const val COUNT = 4
|
||||
|
||||
/** The stream's name in frame metadata. */
|
||||
@JvmStatic
|
||||
fun key(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left"
|
||||
RIGHT -> "right"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The name of the stream's audio tracks in a recording. */
|
||||
@JvmStatic
|
||||
fun trackName(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left temple"
|
||||
RIGHT -> "right temple"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The stream a recording's track or metadata names, or -1. */
|
||||
@JvmStatic
|
||||
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
|
||||
}
|
||||
|
||||
/**
|
||||
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
|
||||
* processing. Live captures, recordings and replays all trade in these.
|
||||
*/
|
||||
class MicChunk(
|
||||
@JvmField val stream: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
|
||||
@JvmField val pcm: ShortArray,
|
||||
@JvmField val frames: Int,
|
||||
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
|
||||
@JvmField val timeMs: Long,
|
||||
) {
|
||||
/** Whether the source reported a direction of arrival with the chunk. */
|
||||
@JvmField var hasDirection = false
|
||||
@JvmField var angleDegrees = 0
|
||||
@JvmField var ssr = 0
|
||||
/** The source's packet counter, or -1 when it has none. */
|
||||
@JvmField var sequence = -1
|
||||
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
|
||||
@JvmField var deviceTickMs = -1L
|
||||
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
|
||||
@JvmField var clockStream = stream
|
||||
/** The source cut the chunk short of what it captured. */
|
||||
@JvmField var truncated = false
|
||||
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
|
||||
@JvmField var lostFramesBefore = 0
|
||||
|
||||
val durationMs: Long get() = frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
|
||||
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
|
||||
* the times they first arrived at, so an implementation that decides from the chunks alone
|
||||
* behaves the same on a replay as it did live.
|
||||
*/
|
||||
interface MicArrayMixer {
|
||||
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
|
||||
fun mix(chunk: MicChunk): MicChunk?
|
||||
|
||||
fun reset()
|
||||
}
|
||||
|
||||
/**
|
||||
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
|
||||
* The temples sample on independent clocks, so their audio is never summed. The left temple
|
||||
* feeds whenever it is streaming; the right one stands in while it is not.
|
||||
*/
|
||||
class SingleTempleMixer : MicArrayMixer {
|
||||
companion object {
|
||||
/** A temple that has delivered nothing for this long no longer counts as streaming. */
|
||||
const val SILENT_MS = 300L
|
||||
}
|
||||
|
||||
private var leftAtMs = Long.MIN_VALUE
|
||||
|
||||
override fun reset() {
|
||||
leftAtMs = Long.MIN_VALUE
|
||||
}
|
||||
|
||||
override fun mix(chunk: MicChunk): MicChunk? {
|
||||
if (chunk.stream == MicStream.LEFT) {
|
||||
leftAtMs = chunk.timeMs
|
||||
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
|
||||
return null
|
||||
}
|
||||
val channels = chunk.channels
|
||||
val mono = ShortArray(chunk.frames)
|
||||
for (frame in 0 until chunk.frames) {
|
||||
var sum = 0
|
||||
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
|
||||
mono[frame] = (sum / channels).toShort()
|
||||
}
|
||||
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
|
||||
out.hasDirection = chunk.hasDirection
|
||||
out.angleDegrees = chunk.angleDegrees
|
||||
out.ssr = chunk.ssr
|
||||
out.deviceTickMs = chunk.deviceTickMs
|
||||
out.clockStream = chunk.stream
|
||||
out.truncated = chunk.truncated
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
|
||||
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
|
||||
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
|
||||
*
|
||||
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
|
||||
* or the count of samples from a source without one) says how far apart its chunks were
|
||||
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
|
||||
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
|
||||
*
|
||||
* Safe to share between the capture thread and whoever finishes the recording.
|
||||
*/
|
||||
class MicRecordingWriter(
|
||||
output: SeekableOutput,
|
||||
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
|
||||
configJson: String?,
|
||||
/** When the recording began (Unix epoch ms). */
|
||||
startedAtMs: Long,
|
||||
platform: ProtocolPlatform = protocolPlatform(),
|
||||
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
|
||||
) {
|
||||
companion object {
|
||||
const val CONFIG_ATTACHMENT = "capture.json"
|
||||
const val META_TRACK = "frames"
|
||||
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
|
||||
const val DISCONTINUITY_MS = 5000L
|
||||
private const val WINDOW_BUCKETS = 8
|
||||
private const val BUCKET_MS = 1000L
|
||||
}
|
||||
|
||||
/**
|
||||
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
|
||||
* since this clock was anchored; the offset from it to arrival time is taken as the least
|
||||
* seen over the last few seconds, which is the least-delayed delivery.
|
||||
*/
|
||||
private class TrackClock(val number: Int, val sampleRate: Int) {
|
||||
private var anchored = false
|
||||
private var clockStream = -1
|
||||
private var tickBase = -1L
|
||||
private var lastTick = -1L
|
||||
private var frames = 0L
|
||||
private val minima = LongArray(WINDOW_BUCKETS)
|
||||
private var bucket = 0
|
||||
private var bucketStartMs = 0L
|
||||
var endMs = Long.MIN_VALUE
|
||||
private set
|
||||
|
||||
fun reset() {
|
||||
anchored = false
|
||||
}
|
||||
|
||||
private fun offset(): Long = minima.min()
|
||||
|
||||
fun stamp(chunk: MicChunk): Long {
|
||||
val arrival = chunk.timeMs
|
||||
val ticked = chunk.deviceTickMs >= 0
|
||||
if (anchored) {
|
||||
// Another clock (the feed moved to the other temple), or this one restarted.
|
||||
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
|
||||
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
|
||||
}
|
||||
var anchorNow = !anchored
|
||||
var source = 0L
|
||||
if (anchored) {
|
||||
frames += chunk.lostFramesBefore
|
||||
source = sourceTime(chunk)
|
||||
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
|
||||
}
|
||||
if (anchorNow) {
|
||||
anchored = true
|
||||
clockStream = chunk.clockStream
|
||||
tickBase = if (ticked) chunk.deviceTickMs else -1L
|
||||
frames = 0
|
||||
minima.fill(Long.MAX_VALUE)
|
||||
bucket = 0
|
||||
bucketStartMs = arrival
|
||||
source = 0
|
||||
}
|
||||
if (arrival - bucketStartMs >= BUCKET_MS) {
|
||||
bucket = (bucket + 1) % WINDOW_BUCKETS
|
||||
minima[bucket] = Long.MAX_VALUE
|
||||
bucketStartMs = arrival
|
||||
}
|
||||
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
|
||||
// Never before the end of the track's previous block.
|
||||
val time = maxOf(source + offset(), endMs)
|
||||
frames += chunk.frames
|
||||
lastTick = chunk.deviceTickMs
|
||||
endMs = time + chunk.durationMs
|
||||
return time
|
||||
}
|
||||
|
||||
private fun sourceTime(chunk: MicChunk): Long =
|
||||
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
private val lock = platform.createLock()
|
||||
private val writer: MkaWriter
|
||||
private val clocks = HashMap<Long, TrackClock>()
|
||||
private var metaTrack = 0
|
||||
private var originMs = Long.MIN_VALUE
|
||||
private var lastTimeMs = 0L
|
||||
private var closed = false
|
||||
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
@Volatile
|
||||
var failed = false
|
||||
private set
|
||||
|
||||
init {
|
||||
val attachments = if (configJson.isNullOrEmpty()) emptyList()
|
||||
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
|
||||
writer = MkaWriter(output, null, startedAtMs, attachments)
|
||||
}
|
||||
|
||||
/** Length of the recording so far, in ms. */
|
||||
val durationMs: Long get() = lock.withLock { writer.durationMs }
|
||||
|
||||
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
|
||||
fun write(chunk: MicChunk) {
|
||||
if (chunk.frames <= 0) return
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
val clock = clockFor(chunk) ?: return
|
||||
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
|
||||
val time = clock.stamp(chunk) - originMs
|
||||
val meta = metadata(chunk)
|
||||
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
|
||||
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
|
||||
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
|
||||
lastTimeMs = maxOf(lastTimeMs, time)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a JSON object as an event at the recording's current position (a capture starting
|
||||
* over with another configuration, say), and has every stream's clock placed afresh by its
|
||||
* next chunk: what follows an event is not continuous with what came before.
|
||||
*/
|
||||
fun event(json: String) {
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
for (clock in clocks.values) clock.reset()
|
||||
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long {
|
||||
lock.withLock {
|
||||
if (!closed) {
|
||||
closed = true
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (error: Throwable) {
|
||||
failed = true
|
||||
logWarn("finishing the recording failed: ${error.message}")
|
||||
}
|
||||
}
|
||||
return writer.durationMs
|
||||
}
|
||||
}
|
||||
|
||||
private fun fail(error: Throwable) {
|
||||
failed = true
|
||||
closed = true
|
||||
logWarn("recording stopped: ${error.message}")
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (ignored: Throwable) {
|
||||
// The file is whatever made it to disk.
|
||||
}
|
||||
}
|
||||
|
||||
private fun ensureMetaTrack(): Boolean {
|
||||
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
|
||||
return metaTrack > 0
|
||||
}
|
||||
|
||||
private fun clockFor(chunk: MicChunk): TrackClock? {
|
||||
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
|
||||
clocks[key]?.let { return it }
|
||||
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
|
||||
ensureMetaTrack()
|
||||
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
|
||||
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
|
||||
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
|
||||
if (number <= 0) {
|
||||
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
|
||||
return null
|
||||
}
|
||||
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
|
||||
}
|
||||
|
||||
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
|
||||
private fun metadata(chunk: MicChunk): String? {
|
||||
if (chunk.stream == MicStream.MIX) return null
|
||||
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
|
||||
val text = StringBuilder(96)
|
||||
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
|
||||
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
|
||||
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
|
||||
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
|
||||
if (chunk.truncated) text.append(",\"trunc\":true")
|
||||
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
|
||||
return text.append('}').toString()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
|
||||
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
|
||||
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
|
||||
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
|
||||
*
|
||||
* Throws IllegalArgumentException when the file has no audio a replay can use.
|
||||
*/
|
||||
class MicRecordingReader(input: SeekableInput) {
|
||||
companion object {
|
||||
const val SAMPLE_RATE = 16000
|
||||
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
|
||||
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
|
||||
}
|
||||
|
||||
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
|
||||
|
||||
private val reader = MkaReader(input)
|
||||
private val streams = HashMap<Int, Int>()
|
||||
private var metaTrack = -1
|
||||
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
|
||||
private val queued = ArrayDeque<MicChunk>()
|
||||
|
||||
val durationMs: Long get() = reader.durationMs
|
||||
|
||||
/** The capture's "capture.json" attachment, or null. */
|
||||
val configJson: String? =
|
||||
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
|
||||
|
||||
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
|
||||
val description: String
|
||||
|
||||
init {
|
||||
try {
|
||||
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
|
||||
for (track in audio) {
|
||||
val stream = MicStream.named(track.name)
|
||||
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
|
||||
}
|
||||
if (streams.isEmpty()) {
|
||||
// Not one of ours: its first usable track stands in for the microphone.
|
||||
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
|
||||
require(first != null) { "no 16-bit PCM audio track" }
|
||||
streams[first.number] = MicStream.MONO
|
||||
}
|
||||
for (number in streams.keys) {
|
||||
val rate = reader.track(number)!!.sampleRate
|
||||
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
|
||||
}
|
||||
metaTrack = reader.tracks.firstOrNull {
|
||||
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
|
||||
}?.number ?: -1
|
||||
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
|
||||
val channels = reader.track(number)!!.channels
|
||||
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
|
||||
}
|
||||
} catch (error: Throwable) {
|
||||
reader.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Continue from the first chunk at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
queued.clear()
|
||||
pendingMeta.fill(null)
|
||||
reader.seekTo(timeMs)
|
||||
}
|
||||
|
||||
/** The next chunk in file order, or null at the end. */
|
||||
fun next(): MicChunk? {
|
||||
while (queued.isEmpty()) {
|
||||
val block = reader.next() ?: return null
|
||||
if (block.track == metaTrack) {
|
||||
noteMeta(block)
|
||||
continue
|
||||
}
|
||||
val stream = streams[block.track] ?: continue
|
||||
val track = reader.track(block.track) ?: continue
|
||||
queueChunks(block, stream, track)
|
||||
}
|
||||
return queued.removeFirst()
|
||||
}
|
||||
|
||||
fun close() = reader.close()
|
||||
|
||||
private fun noteMeta(block: MkaBlock) {
|
||||
try {
|
||||
val fields = Json.parseObject(block.data.decodeToString())
|
||||
val stream = MicStream.named(fields["s"].asString() ?: return)
|
||||
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
|
||||
} catch (ignored: Throwable) {
|
||||
// An event or a frame we can't read: the audio plays without it.
|
||||
}
|
||||
}
|
||||
|
||||
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
|
||||
val sourceChannels = track.channels
|
||||
val totalFrames = block.data.size / (2 * sourceChannels)
|
||||
if (totalFrames <= 0) return
|
||||
// The mixed microphone is one channel however a foreign file stored it.
|
||||
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
|
||||
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
|
||||
pendingMeta[stream] = null
|
||||
var start = 0
|
||||
while (start < totalFrames) {
|
||||
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
|
||||
val pcm = ShortArray(frames * channels)
|
||||
for (frame in 0 until frames) {
|
||||
val at = (start + frame) * sourceChannels * 2
|
||||
if (channels == sourceChannels) {
|
||||
for (channel in 0 until channels) {
|
||||
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
}
|
||||
} else {
|
||||
var sum = 0
|
||||
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
pcm[frame] = (sum / sourceChannels).toShort()
|
||||
}
|
||||
}
|
||||
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
|
||||
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
|
||||
queued.addLast(chunk)
|
||||
start += frames
|
||||
}
|
||||
}
|
||||
|
||||
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
|
||||
chunk.sequence = fields["seq"].asInt() ?: -1
|
||||
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
|
||||
val angle = fields["angle"].asInt()
|
||||
if (angle != null) {
|
||||
chunk.hasDirection = true
|
||||
chunk.angleDegrees = angle
|
||||
chunk.ssr = fields["ssr"].asInt() ?: 0
|
||||
}
|
||||
chunk.truncated = fields["trunc"].asBoolean() ?: false
|
||||
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
|
||||
}
|
||||
}
|
||||
+393
-46
@@ -35,6 +35,12 @@ interface VoiceCaptureHost {
|
||||
|
||||
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
|
||||
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
|
||||
|
||||
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
|
||||
fun micArrayLink(): MicArrayLink? = null
|
||||
|
||||
/** Opens a recording to replay; throws when it cannot be read. */
|
||||
fun openRecording(path: String): SeekableInput = openFileInput(path)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -44,6 +50,12 @@ interface VoiceCaptureHost {
|
||||
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
|
||||
* worker thread (started through the shared [startThread]); listener callbacks go through the
|
||||
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
|
||||
*
|
||||
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
|
||||
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
|
||||
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
|
||||
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
|
||||
* replay of the recording run the same processing again.
|
||||
*/
|
||||
class VoiceCaptureSession(
|
||||
private val host: VoiceCaptureHost,
|
||||
@@ -95,6 +107,19 @@ class VoiceCaptureSession(
|
||||
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
|
||||
/** Samples per G2 packet; the decode buffer the worker reuses. */
|
||||
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
|
||||
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
|
||||
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
|
||||
const val ARRAY_CHECK_MS = 1000L
|
||||
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
|
||||
const val ARRAY_TEMPLE_SILENT_MS = 5000L
|
||||
const val ARRAY_WAITING_MS = 2000L
|
||||
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
|
||||
// A replay skips stretches of the recording with no audio longer than this
|
||||
// (a recording made across a disconnect) instead of sitting through them.
|
||||
const val REPLAY_MAX_GAP_MS = 2000L
|
||||
const val REPLAY_POSITION_INTERVAL_MS = 1000L
|
||||
const val REPLAY_WAIT_SLICE_MS = 20L
|
||||
const val REPLAY_FINISHED_STATUS = "Replay finished."
|
||||
|
||||
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
|
||||
@JvmStatic
|
||||
@@ -120,6 +145,8 @@ class VoiceCaptureSession(
|
||||
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
private enum class Source { G2, PHONE, ARRAY, REPLAY }
|
||||
|
||||
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
|
||||
val arm: String = arm ?: "?"
|
||||
}
|
||||
@@ -137,11 +164,33 @@ class VoiceCaptureSession(
|
||||
@Volatile
|
||||
private var workerThreadName: String? = null
|
||||
// Capture from the phone's own microphone instead of the G2 over BLE
|
||||
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
|
||||
// (preview-only mode). Latched into activeSource at start() (under `lock`)
|
||||
// so a mid-session setter call can't switch pipelines underneath the worker.
|
||||
@Volatile
|
||||
private var usePhoneMic = false
|
||||
private var activePhoneMic = false
|
||||
// What the next capture reads (see start()); latched into activeSource and its
|
||||
// companions under `lock`, like usePhoneMic.
|
||||
@Volatile
|
||||
private var micArrayConfigure: ByteArray? = null
|
||||
@Volatile
|
||||
private var replayPath: String? = null
|
||||
@Volatile
|
||||
private var replayStartMs = 0L
|
||||
private var activeSource = Source.G2
|
||||
private var activeConfigure: ByteArray? = null
|
||||
private var activeReplayPath: String? = null
|
||||
private var activeReplayStartMs = 0L
|
||||
// Replay controls, picked up by the worker between chunks.
|
||||
@Volatile
|
||||
private var replaySeekMs = -1L
|
||||
@Volatile
|
||||
private var replayPaused = false
|
||||
@Volatile
|
||||
private var recorder: MicRecordingWriter? = null
|
||||
@Volatile
|
||||
private var mixer: MicArrayMixer = SingleTempleMixer()
|
||||
private var lastMissingPackets = 0L
|
||||
private var arrayRateWarned = false
|
||||
private var mode = Mode.CLOUD
|
||||
@Volatile
|
||||
private var onboardModelKind = VoiceModelKind.MOONSHINE
|
||||
@@ -193,6 +242,47 @@ class VoiceCaptureSession(
|
||||
this.usePhoneMic = usePhoneMic
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture the next glasses session from the temples' own microphones instead of the stock
|
||||
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
|
||||
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
|
||||
}
|
||||
|
||||
/** How the temples' chunks become the mono feed that is transcribed. */
|
||||
fun setMicArrayMixer(mixer: MicArrayMixer) {
|
||||
this.mixer = mixer
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next capture play the recording at [path] from [startMs] in real time, in place
|
||||
* of any microphone; null goes back to the microphones. A replay needs no glasses.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
this.replayPath = path
|
||||
this.replayStartMs = maxOf(0L, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
this.replaySeekMs = maxOf(0L, positionMs)
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
this.replayPaused = paused
|
||||
}
|
||||
|
||||
/**
|
||||
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
|
||||
* also mid-capture; the caller finishes the recording. A replay is never recorded.
|
||||
*/
|
||||
fun setRecorder(recorder: MicRecordingWriter?) {
|
||||
this.recorder = recorder
|
||||
}
|
||||
|
||||
fun setOnboardModelKind(kind: String?) {
|
||||
this.onboardModelKind = parseModelKind(kind)
|
||||
}
|
||||
@@ -258,13 +348,26 @@ class VoiceCaptureSession(
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
if (!usePhoneMic && !host.isG2SessionReady()) {
|
||||
val path = replayPath
|
||||
val configure = micArrayConfigure
|
||||
val source = when {
|
||||
path != null -> Source.REPLAY
|
||||
usePhoneMic -> Source.PHONE
|
||||
configure != null -> Source.ARRAY
|
||||
else -> Source.G2
|
||||
}
|
||||
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
|
||||
emitStatus("Voice control needs an active G2 connection.")
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
mode = parseMode(requestedMode)
|
||||
activePhoneMic = usePhoneMic
|
||||
activeSource = source
|
||||
activeConfigure = configure
|
||||
activeReplayPath = path
|
||||
activeReplayStartMs = replayStartMs
|
||||
replaySeekMs = -1L
|
||||
replayPaused = false
|
||||
started = true
|
||||
audioStarted = false
|
||||
val finished = Latch(1, platform)
|
||||
@@ -281,21 +384,22 @@ class VoiceCaptureSession(
|
||||
*/
|
||||
fun isCapturing(): Boolean {
|
||||
var audioUp = false
|
||||
var phoneMic = false
|
||||
var source = Source.G2
|
||||
lock.withLock {
|
||||
if (!started) {
|
||||
return false
|
||||
}
|
||||
audioUp = audioStarted
|
||||
phoneMic = activePhoneMic
|
||||
source = activeSource
|
||||
}
|
||||
if (!audioUp) {
|
||||
// The worker is still bringing the mic up; report it as running so
|
||||
// a concurrent request shares it instead of restarting it.
|
||||
return true
|
||||
}
|
||||
if (phoneMic) {
|
||||
// The phone mic has no session to lose the enable to; it runs until stop().
|
||||
if (source == Source.PHONE || source == Source.REPLAY) {
|
||||
// Neither has a glasses session to lose the enable to: the phone mic runs
|
||||
// until stop(), a replay until its recording ends.
|
||||
return true
|
||||
}
|
||||
return host.isG2AudioCaptureActive()
|
||||
@@ -317,7 +421,7 @@ class VoiceCaptureSession(
|
||||
finished = workerFinished
|
||||
workerName = workerThreadName
|
||||
}
|
||||
stopG2Audio()
|
||||
stopSource()
|
||||
audioQueue.wake()
|
||||
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
|
||||
finished.await(1500)
|
||||
@@ -356,28 +460,62 @@ class VoiceCaptureSession(
|
||||
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
|
||||
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
|
||||
verifyCount = 0
|
||||
if (activePhoneMic) {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
mixer.reset()
|
||||
lastMissingPackets = 0
|
||||
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
|
||||
when (activeSource) {
|
||||
Source.PHONE -> {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
}
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
Source.REPLAY -> {
|
||||
val reader = try {
|
||||
MicRecordingReader(host.openRecording(activeReplayPath!!))
|
||||
} catch (error: Throwable) {
|
||||
emitStatus("Could not open the recording: " + error.message)
|
||||
emitReplay("error", 0, 0)
|
||||
return
|
||||
}
|
||||
try {
|
||||
lock.withLock { audioStarted = true }
|
||||
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
|
||||
emitStatus(listening)
|
||||
processReplay(reader)
|
||||
} finally {
|
||||
reader.close()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
Source.ARRAY -> {
|
||||
val link = host.micArrayLink()
|
||||
val configure = activeConfigure
|
||||
if (link == null || configure == null || !startArray(link, configure)) {
|
||||
emitStatus("Could not start the glasses microphone array.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processArray(link, configure, listening)
|
||||
}
|
||||
Source.G2 -> {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processG2Audio()
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
processG2Audio()
|
||||
}
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
@@ -392,7 +530,7 @@ class VoiceCaptureSession(
|
||||
logError("Voice control failed", error)
|
||||
emitStatus("Voice control failed: " + error.message)
|
||||
} finally {
|
||||
stopG2Audio()
|
||||
stopSource()
|
||||
releaseRecognizer()
|
||||
releaseDecoder()
|
||||
lock.withLock {
|
||||
@@ -432,23 +570,214 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += count.toLong()
|
||||
// The beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker for
|
||||
// every consumer, recognition included); the spectral noise
|
||||
// suppressor then cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
val angleDegrees = currentDecoder.lastAngleDegrees
|
||||
val ssr = currentDecoder.lastSsr
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr)
|
||||
maybeEmitAudioStats(false)
|
||||
continue
|
||||
}
|
||||
processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta)
|
||||
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
|
||||
chunk.hasDirection = currentDecoder.hasFrameMeta
|
||||
chunk.angleDegrees = currentDecoder.lastAngleDegrees
|
||||
chunk.ssr = currentDecoder.lastSsr
|
||||
// Packets the decoder found missing ahead of this one: a gap in a recording.
|
||||
val missing = currentDecoder.missingPackets
|
||||
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
|
||||
lastMissingPackets = missing
|
||||
handleMonoChunk(chunk)
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
|
||||
* unless the beam filter drops it. The filter drops chunks whose firmware
|
||||
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
|
||||
* every consumer, recognition included); the spectral noise suppressor then cleans what
|
||||
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
*/
|
||||
private fun handleMonoChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
|
||||
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
|
||||
return
|
||||
}
|
||||
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
|
||||
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
|
||||
* The beam filter does not apply: a temple's bearing is along its own pair, not the
|
||||
* head-frame direction the filter is aimed in.
|
||||
*/
|
||||
private fun handleArrayChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
val feed = mixer.mix(chunk) ?: return
|
||||
recordChunk(feed)
|
||||
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
|
||||
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
|
||||
arrayRateWarned = true
|
||||
return
|
||||
}
|
||||
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
|
||||
}
|
||||
|
||||
private fun recordChunk(chunk: MicChunk) {
|
||||
if (activeSource != Source.REPLAY) recorder?.write(chunk)
|
||||
}
|
||||
|
||||
private fun handleChunk(chunk: MicChunk) {
|
||||
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
|
||||
}
|
||||
|
||||
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
|
||||
resetAudioStats()
|
||||
audioQueue.clear()
|
||||
if (!startArrayForwarding(link)) return false
|
||||
return link.sendControl(configure, "configure", true, true)
|
||||
}
|
||||
|
||||
private fun startArrayForwarding(link: MicArrayLink): Boolean =
|
||||
link.startForwarding(object : FaceclawAudioPacketListener {
|
||||
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
|
||||
queueAudioPacket(data, arm, arrivalMs)
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
|
||||
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
|
||||
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
|
||||
*/
|
||||
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
|
||||
val startedAt = platform.elapsedRealtimeMs()
|
||||
var checkedAt = startedAt
|
||||
var renewedAt = startedAt
|
||||
// Left, right.
|
||||
val armedAt = longArrayOf(startedAt, startedAt)
|
||||
val frameAt = longArrayOf(startedAt, startedAt)
|
||||
var waiting = false
|
||||
while (started) {
|
||||
val packet = takeAudioPacket()
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (packet != null) {
|
||||
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
|
||||
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
|
||||
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
|
||||
// Anything else is a stock-format packet from before arming took effect.
|
||||
if (frame != null && pcm != null) {
|
||||
frameAt[side] = now
|
||||
val frames = pcm.size / frame.channels
|
||||
decodedSamples += frames.toLong()
|
||||
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
|
||||
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
|
||||
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
|
||||
chunk.angleDegrees = frame.angleDegrees
|
||||
chunk.ssr = frame.ssr
|
||||
chunk.sequence = frame.sequence
|
||||
chunk.deviceTickMs = frame.deviceTickMs
|
||||
chunk.truncated = frame.truncated
|
||||
handleArrayChunk(chunk)
|
||||
}
|
||||
}
|
||||
if (now - checkedAt >= ARRAY_CHECK_MS) {
|
||||
checkedAt = now
|
||||
if (!link.isForwarding()) {
|
||||
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
|
||||
logInfo("mic array forwarding re-established")
|
||||
armedAt.fill(now)
|
||||
renewedAt = now
|
||||
}
|
||||
} else {
|
||||
if (now - renewedAt >= ARRAY_RENEW_MS) {
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
|
||||
renewedAt = now
|
||||
}
|
||||
for (side in 0..1) {
|
||||
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
|
||||
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
|
||||
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
|
||||
armedAt[side] = now
|
||||
}
|
||||
}
|
||||
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
|
||||
if (silent != waiting) {
|
||||
waiting = silent
|
||||
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
|
||||
}
|
||||
}
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
|
||||
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
|
||||
* Jumps and pauses asked for from outside take effect between chunks.
|
||||
*/
|
||||
private fun processReplay(reader: MicRecordingReader) {
|
||||
val duration = reader.durationMs
|
||||
var position = minOf(activeReplayStartMs, duration)
|
||||
reader.seekTo(position)
|
||||
emitReplay("start", position, duration)
|
||||
var reportedAt = position
|
||||
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
|
||||
var anchorWall = -1L
|
||||
var anchorFile = 0L
|
||||
var pending: MicChunk? = null
|
||||
while (started) {
|
||||
val seek = replaySeekMs
|
||||
if (seek >= 0) {
|
||||
replaySeekMs = -1L
|
||||
// What played so far is a finished utterance; the rest starts clean.
|
||||
finishReplayedUtterance()
|
||||
position = minOf(seek, duration)
|
||||
reader.seekTo(position)
|
||||
pending = null
|
||||
anchorWall = -1L
|
||||
reportedAt = position
|
||||
emitReplay("seek", position, duration)
|
||||
}
|
||||
if (replayPaused) {
|
||||
anchorWall = -1L
|
||||
sleepMs(REPLAY_WAIT_SLICE_MS)
|
||||
continue
|
||||
}
|
||||
val chunk = pending ?: reader.next()
|
||||
if (chunk == null) {
|
||||
emitReplay("end", duration, duration)
|
||||
emitStatus(REPLAY_FINISHED_STATUS)
|
||||
return
|
||||
}
|
||||
pending = chunk
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
|
||||
anchorWall = now
|
||||
anchorFile = chunk.timeMs
|
||||
}
|
||||
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
|
||||
if (wait > 0) {
|
||||
// Short slices, so a stop, jump or pause is noticed promptly.
|
||||
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
|
||||
continue
|
||||
}
|
||||
pending = null
|
||||
position = chunk.timeMs
|
||||
decodedSamples += chunk.frames.toLong()
|
||||
handleChunk(chunk)
|
||||
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
|
||||
reportedAt = position
|
||||
emitReplay("position", position, duration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
|
||||
private fun finishReplayedUtterance() {
|
||||
mixer.reset()
|
||||
endpointDetector.reset()
|
||||
suppressor?.reset()
|
||||
if (mode != Mode.ONBOARD) return
|
||||
decodeTranscript(true)
|
||||
resetTranscriptState()
|
||||
lastTranscript = ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
|
||||
* decode and the beam filter. hasFrameMeta is false for the phone mic,
|
||||
@@ -495,7 +824,7 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += read.toLong()
|
||||
processPcmChunk(pcm, read, 0, 0, false)
|
||||
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -681,6 +1010,11 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
|
||||
}
|
||||
|
||||
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
|
||||
}
|
||||
|
||||
private fun emitSpeechEnd() {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onSpeechEnd() }
|
||||
@@ -691,8 +1025,21 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onStopped(captureId) }
|
||||
}
|
||||
|
||||
private fun stopG2Audio() {
|
||||
host.g2AudioSource()?.stop()
|
||||
/**
|
||||
* Release what the running capture's source holds on the glasses. Runs on stop() and again
|
||||
* as the worker winds down (it may have started the source after stop() looked), so
|
||||
* everything here is safe to repeat.
|
||||
*/
|
||||
private fun stopSource() {
|
||||
val source = lock.withLock { activeSource }
|
||||
when (source) {
|
||||
Source.G2 -> host.g2AudioSource()?.stop()
|
||||
Source.ARRAY -> host.micArrayLink()?.let { link ->
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
|
||||
link.stopForwarding()
|
||||
}
|
||||
Source.PHONE, Source.REPLAY -> {}
|
||||
}
|
||||
maybeEmitAudioStats(true)
|
||||
}
|
||||
|
||||
@@ -711,7 +1058,7 @@ class VoiceCaptureSession(
|
||||
if (!started || data == null) {
|
||||
return
|
||||
}
|
||||
if ("L" != arm) {
|
||||
if ("L" != arm && activeSource != Source.ARRAY) {
|
||||
wrongArmPackets++
|
||||
}
|
||||
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
|
||||
|
||||
@@ -67,6 +67,24 @@ interface PacketAudioSource {
|
||||
fun isActive(): Boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
|
||||
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
|
||||
* stream uses, while it holds a configuration that the phone keeps renewing.
|
||||
*/
|
||||
interface MicArrayLink {
|
||||
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
|
||||
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
|
||||
|
||||
fun stopForwarding()
|
||||
|
||||
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
|
||||
fun isForwarding(): Boolean
|
||||
|
||||
/** Queue a control record for the chosen temples; false when nothing was queued. */
|
||||
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
|
||||
}
|
||||
|
||||
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
|
||||
interface PcmAudioSource {
|
||||
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
|
||||
|
||||
+9
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
|
||||
|
||||
/**
|
||||
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
|
||||
* "start" once the file is open, "position" about once per second of it played, "seek" after
|
||||
* a jump took effect (the transcript of what played before it has been emitted as final),
|
||||
* "end" when the file ran out, and "error" when it could not be opened (the status says
|
||||
* why). Positions are milliseconds into the file.
|
||||
*/
|
||||
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import platform.Foundation.NSData
|
||||
|
||||
/**
|
||||
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
|
||||
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
|
||||
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
|
||||
*/
|
||||
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
val failed: Boolean get() = writer.failed
|
||||
|
||||
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
|
||||
fun append(pcm16le: NSData) {
|
||||
val bytes = pcm16le.byteArray()
|
||||
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
|
||||
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long = writer.finish()
|
||||
}
|
||||
|
||||
object IosMicRecordings {
|
||||
/**
|
||||
* A recorder writing a new file at [path], or null when the file cannot be created.
|
||||
* [configJson] describes the capture and is stored in the file.
|
||||
*/
|
||||
fun open(path: String, configJson: String?): IosMicRecorder? =
|
||||
try {
|
||||
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
|
||||
} catch (error: Throwable) {
|
||||
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
|
||||
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput {
|
||||
val file = fopen(path, "wb+") ?: error("Cannot write $path")
|
||||
return object : SeekableOutput {
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
private fun put(bytes: ByteArray, offset: Int, length: Int) {
|
||||
if (length == 0) return
|
||||
bytes.usePinned {
|
||||
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
put(bytes, offset, length)
|
||||
size += length
|
||||
// Each write is a complete element; hand it to the OS so a crash keeps it.
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
|
||||
put(bytes, 0, bytes.size)
|
||||
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput {
|
||||
val file = fopen(path, "rb") ?: error("Cannot read $path")
|
||||
val fileSize = fileLength(path)
|
||||
return object : SeekableInput {
|
||||
override val length = fileSize
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
|
||||
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ function harness() {
|
||||
static open(name) {
|
||||
recordingAttempts++;
|
||||
if (failRecordings) return null;
|
||||
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.mka`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||
}
|
||||
append(bytes) { this.pcm.push(...bytes); }
|
||||
close() { this.closed = true; }
|
||||
@@ -49,7 +49,7 @@ function harness() {
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer
|
||||
: name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
: name === './mic-recorder.ios' ? { MicRecorder: FakeRecorder } : {} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import java.io.File
|
||||
import kotlin.math.PI
|
||||
import kotlin.math.sin
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class MicRecordingFileTest {
|
||||
/**
|
||||
* A capture recorded through the real file adapters and read back. Set FACECLAW_MKA_SAMPLE
|
||||
* to a path to keep the file, for checking it against other Matroska tools (ffprobe, mpv).
|
||||
*/
|
||||
@Test
|
||||
fun recordsACaptureToAFileThatReadsBack() {
|
||||
val kept = System.getenv("FACECLAW_MKA_SAMPLE")
|
||||
val file = if (kept.isNullOrEmpty()) File.createTempFile("capture", ".mka") else File(kept)
|
||||
try {
|
||||
val writer = MicRecordingWriter(openFileOutput(file.path), "{\"mics\":4,\"app\":\"test\"}", 1_700_000_000_000)
|
||||
val mixer = SingleTempleMixer()
|
||||
// Five seconds: each temple streams a stereo pair of tones on its own tick.
|
||||
for (index in 0 until 100) {
|
||||
for (side in 0..1) {
|
||||
val pcm = ShortArray(1600) {
|
||||
val frame = index * 800 + it / 2
|
||||
val hz = if (side == 0) 440.0 else 660.0
|
||||
(sin(2 * PI * hz * frame / 16000) * (if (it % 2 == 0) 8000 else 4000)).toInt().toShort()
|
||||
}
|
||||
val chunk = MicChunk(if (side == 0) MicStream.LEFT else MicStream.RIGHT, 2, 16000, pcm, 800, 10_000 + index * 50L + side * 7)
|
||||
chunk.sequence = index
|
||||
chunk.deviceTickMs = (if (side == 0) 500_000L else 123_456L) + index * 50L
|
||||
chunk.hasDirection = true
|
||||
chunk.angleDegrees = if (side == 0) -30 else 25
|
||||
chunk.ssr = 12
|
||||
writer.write(chunk)
|
||||
mixer.mix(chunk)?.let { writer.write(it) }
|
||||
}
|
||||
}
|
||||
// The right temple's chunks arrive 7 ms behind the left's.
|
||||
assertEquals(5007L, writer.finish())
|
||||
|
||||
val reader = MicRecordingReader(openFileInput(file.path))
|
||||
assertEquals(5007L, reader.durationMs)
|
||||
assertEquals("left temple 2ch + right temple 2ch", reader.description)
|
||||
assertEquals("{\"mics\":4,\"app\":\"test\"}", reader.configJson)
|
||||
var left = 0
|
||||
var right = 0
|
||||
while (true) {
|
||||
val chunk = reader.next() ?: break
|
||||
if (chunk.stream == MicStream.LEFT) left++ else right++
|
||||
assertEquals(12, chunk.ssr)
|
||||
}
|
||||
assertEquals(100, left)
|
||||
assertEquals(100, right)
|
||||
reader.close()
|
||||
// Audio (three tracks) plus the frame metadata.
|
||||
assertTrue(file.length() > 100 * 2 * 3200 + 100 * 1600)
|
||||
} finally {
|
||||
if (kept.isNullOrEmpty()) file.delete()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFailsWith
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNotNull
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
/** A file held in memory: written through [SeekableOutput], read back through [input]. */
|
||||
internal class MemoryFile : SeekableOutput {
|
||||
var bytes = ByteArray(0)
|
||||
var closed = false
|
||||
override val size: Long get() = bytes.size.toLong()
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
this.bytes += bytes.copyOfRange(offset, offset + length)
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
bytes.copyInto(this.bytes, position.toInt())
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
closed = true
|
||||
}
|
||||
|
||||
/** The file as it stands, or its first [length] bytes. */
|
||||
fun input(length: Int = bytes.size): SeekableInput {
|
||||
val snapshot = bytes.copyOf(length)
|
||||
return object : SeekableInput {
|
||||
override val length = snapshot.size.toLong()
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
val count = minOf(length.toLong(), maxOf(0L, snapshot.size - position)).toInt()
|
||||
snapshot.copyInto(buffer, offset, position.toInt(), position.toInt() + count)
|
||||
return count
|
||||
}
|
||||
|
||||
override fun close() {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun pcmBytes(frames: Int, channels: Int, value: Int): ByteArray =
|
||||
AudioSegmentation.pcm16ToLittleEndian(ShortArray(frames * channels) { value.toShort() }, 0, frames * channels)
|
||||
|
||||
private fun chunk(stream: Int, timeMs: Long, value: Int, channels: Int = 1, frames: Int = 800): MicChunk =
|
||||
MicChunk(stream, channels, 16000, ShortArray(frames * channels) { (value + it % channels).toShort() }, frames, timeMs)
|
||||
|
||||
private fun drain(reader: MicRecordingReader): List<MicChunk> {
|
||||
val out = ArrayList<MicChunk>()
|
||||
while (true) out.add(reader.next() ?: break)
|
||||
return out
|
||||
}
|
||||
|
||||
class MatroskaTest {
|
||||
@Test
|
||||
fun roundTripsTracksBlocksAndAttachments() {
|
||||
val file = MemoryFile()
|
||||
val writer = MkaWriter(file, "session", 1_700_000_000_000, listOf(MkaAttachment("capture.json", "application/json", "{\"a\":1}".encodeToByteArray())))
|
||||
val temple = writer.addAudioTrack("left temple", 16000, 2, false)
|
||||
val text = writer.addTextTrack("frames")
|
||||
for (index in 0 until 30) {
|
||||
writer.writeText(text, index * 50L, 50, "{\"n\":$index}")
|
||||
writer.writeAudio(temple, index * 50L, pcmBytes(800, 2, index), 0, 3200)
|
||||
}
|
||||
// A track that first appears mid-file, and is the one players should pick.
|
||||
val mix = writer.addAudioTrack("mix", 16000, 1, true)
|
||||
for (index in 30 until 60) {
|
||||
writer.writeAudio(temple, index * 50L, pcmBytes(800, 2, index), 0, 3200)
|
||||
writer.writeAudio(mix, index * 50L, pcmBytes(800, 1, -index), 0, 1600)
|
||||
}
|
||||
assertEquals(3000L, writer.durationMs)
|
||||
writer.finish()
|
||||
assertTrue(file.closed)
|
||||
|
||||
val reader = MkaReader(file.input())
|
||||
// Listed with the preferred audio track first, then the others, then text; numbers are as added.
|
||||
assertEquals(listOf("mix", "left temple", "frames"), reader.tracks.map { it.name })
|
||||
assertEquals(listOf(mix, temple, text), reader.tracks.map { it.number })
|
||||
val templeInfo = reader.track(temple)!!
|
||||
assertTrue(templeInfo.isPcm16)
|
||||
assertEquals(2, templeInfo.channels)
|
||||
assertEquals(16000, templeInfo.sampleRate)
|
||||
assertFalse(reader.track(text)!!.isPcm16)
|
||||
assertEquals("{\"a\":1}", reader.attachments.single().data.decodeToString())
|
||||
assertEquals("application/json", reader.attachments.single().mimeType)
|
||||
assertEquals(3000L, reader.durationMs)
|
||||
|
||||
val blocks = ArrayList<MkaBlock>()
|
||||
while (true) blocks.add(reader.next() ?: break)
|
||||
assertEquals(30 + 60 + 30, blocks.size)
|
||||
assertEquals(text, blocks[0].track)
|
||||
assertEquals("{\"n\":0}", blocks[0].data.decodeToString())
|
||||
assertEquals(50L, blocks[0].durationMs)
|
||||
val templeBlocks = blocks.filter { it.track == temple }
|
||||
assertEquals((0 until 60).map { it * 50L }, templeBlocks.map { it.timeMs })
|
||||
assertEquals(pcmBytes(800, 2, 59).toList(), templeBlocks.last().data.toList())
|
||||
assertEquals(1500L, blocks.first { it.track == mix }.timeMs)
|
||||
|
||||
reader.seekTo(2000)
|
||||
val afterSeek = ArrayList<MkaBlock>()
|
||||
while (true) afterSeek.add(reader.next() ?: break)
|
||||
assertEquals(2000L, afterSeek.first().timeMs)
|
||||
assertEquals(40, afterSeek.size)
|
||||
reader.seekTo(99_000)
|
||||
assertNull(reader.next())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun recordingCutShortIsReadableUpToItsLastCompleteBlock() {
|
||||
val file = MemoryFile()
|
||||
val writer = MkaWriter(file, null, 0, emptyList())
|
||||
val track = writer.addAudioTrack("mic", 16000, 1, true)
|
||||
// 2.5 s: clusters of a second each reach the file as they fill; the last half-second is still buffered.
|
||||
for (index in 0 until 50) writer.writeAudio(track, index * 50L, pcmBytes(800, 1, index), 0, 1600)
|
||||
val whole = MkaReader(file.input())
|
||||
assertEquals("mic", whole.tracks.single().name)
|
||||
var count = 0
|
||||
while (whole.next() != null) count++
|
||||
assertEquals(40, count)
|
||||
assertEquals(2000L, whole.durationMs)
|
||||
// A crash mid-write tears the last cluster: its complete blocks still read.
|
||||
val torn = MkaReader(file.input(file.bytes.size - 1000))
|
||||
count = 0
|
||||
while (torn.next() != null) count++
|
||||
assertEquals(39, count)
|
||||
assertEquals(1950L, torn.durationMs)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rejectsOtherFiles() {
|
||||
val wav = MemoryFile()
|
||||
wav.write(BinaryEncoding.wavHeader(0, 16000, 1), 0, 44)
|
||||
assertFailsWith<IllegalArgumentException> { MkaReader(wav.input()) }
|
||||
assertFailsWith<IllegalArgumentException> { MkaReader(MemoryFile().input()) }
|
||||
}
|
||||
|
||||
@Test
|
||||
fun paddedRegionsFillTheirReservedSpaceWithAVoidElement() {
|
||||
assertNull(EbmlWriter.padded(ByteArray(10), 9))
|
||||
assertNull(EbmlWriter.padded(ByteArray(10), 11))
|
||||
assertEquals(listOf(1, 2, 0xEC, 0x80), EbmlWriter.padded(byteArrayOf(1, 2), 4)!!.map { it.toInt() and 0xff })
|
||||
val long = EbmlWriter.padded(ByteArray(0), 300)!!
|
||||
assertEquals(300, long.size)
|
||||
// An eight-byte size field: 300 - 9 = 0x123.
|
||||
assertEquals(listOf(0xEC, 0x01, 0, 0, 0, 0, 0, 0x01, 0x23), long.take(9).map { it.toInt() and 0xff })
|
||||
}
|
||||
}
|
||||
|
||||
class MicRecordingTest {
|
||||
private val warnings = mutableListOf<String>()
|
||||
|
||||
private fun recorder(file: MemoryFile, config: String? = "{\"mics\":4}") =
|
||||
MicRecordingWriter(file, config, 1_700_000_000_000, testPlatform()) { warnings.add(it) }
|
||||
|
||||
@Test
|
||||
fun recordsEveryStreamWithItsMetadataAndReplaysTheSources() {
|
||||
val file = MemoryFile()
|
||||
val writer = recorder(file)
|
||||
for (index in 0 until 4) {
|
||||
val left = chunk(MicStream.LEFT, 5000 + index * 50L, 100 + index, channels = 2)
|
||||
left.sequence = 7 + index
|
||||
left.deviceTickMs = 900_000 + index * 50L
|
||||
left.hasDirection = true
|
||||
left.angleDegrees = -12
|
||||
left.ssr = 40
|
||||
left.truncated = index == 3
|
||||
writer.write(left)
|
||||
val feed = SingleTempleMixer().mix(left)!!
|
||||
writer.write(feed)
|
||||
val right = chunk(MicStream.RIGHT, 5010 + index * 50L, 200 + index, channels = 2)
|
||||
right.sequence = 3 + index
|
||||
right.deviceTickMs = 40_000 + index * 50L
|
||||
writer.write(right)
|
||||
}
|
||||
// The capture went back to the mixed stream part-way through.
|
||||
writer.event("{\"event\":\"capture\",\"mics\":1}")
|
||||
val mono = chunk(MicStream.MONO, 5200, 55)
|
||||
mono.hasDirection = true
|
||||
mono.angleDegrees = 30
|
||||
mono.ssr = 9
|
||||
writer.write(mono)
|
||||
assertEquals(250L, writer.finish())
|
||||
assertFalse(writer.failed)
|
||||
assertTrue(warnings.isEmpty(), warnings.toString())
|
||||
|
||||
val mka = MkaReader(file.input())
|
||||
// What a player picks by default is a mixed track.
|
||||
assertEquals(listOf("mix", "mic", "left temple", "right temple", "frames"), mka.tracks.map { it.name })
|
||||
|
||||
val reader = MicRecordingReader(file.input())
|
||||
assertEquals("{\"mics\":4}", reader.configJson)
|
||||
assertEquals("mic + left temple 2ch + right temple 2ch", reader.description)
|
||||
assertEquals(250L, reader.durationMs)
|
||||
val chunks = drain(reader)
|
||||
// The mix is derived again on a replay, so only the sources come back.
|
||||
assertEquals(4, chunks.count { it.stream == MicStream.LEFT })
|
||||
assertEquals(4, chunks.count { it.stream == MicStream.RIGHT })
|
||||
assertEquals(1, chunks.count { it.stream == MicStream.MONO })
|
||||
val left = chunks.filter { it.stream == MicStream.LEFT }
|
||||
assertEquals(listOf(0L, 50L, 100L, 150L), left.map { it.timeMs })
|
||||
assertEquals(listOf(7, 8, 9, 10), left.map { it.sequence })
|
||||
assertEquals(900_050L, left[1].deviceTickMs)
|
||||
assertEquals(2, left[0].channels)
|
||||
assertEquals(listOf<Short>(100, 101, 100, 101), left[0].pcm.take(4))
|
||||
assertTrue(left.all { it.hasDirection && it.angleDegrees == -12 && it.ssr == 40 })
|
||||
assertEquals(listOf(false, false, false, true), left.map { it.truncated })
|
||||
val right = chunks.filter { it.stream == MicStream.RIGHT }
|
||||
assertEquals(listOf(10L, 60L, 110L, 160L), right.map { it.timeMs })
|
||||
assertFalse(right[0].hasDirection)
|
||||
val replayedMono = chunks.single { it.stream == MicStream.MONO }
|
||||
assertEquals(200L, replayedMono.timeMs)
|
||||
assertEquals(30, replayedMono.angleDegrees)
|
||||
assertEquals(800, replayedMono.frames)
|
||||
|
||||
reader.seekTo(100)
|
||||
assertEquals(100L, reader.next()!!.timeMs)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun chunksHeldUpInDeliveryLandWhereTheyWereCaptured() {
|
||||
val file = MemoryFile()
|
||||
val writer = recorder(file, null)
|
||||
// A temple's ticks say when each chunk was captured. The first ten arrive as captured;
|
||||
// the link then stalls for a second and delivers the next twenty at once.
|
||||
for (index in 0 until 30) {
|
||||
val arrival = if (index < 10) 1000 + index * 50L else maxOf(2500L, 1000 + index * 50L)
|
||||
val chunk = chunk(MicStream.LEFT, arrival, index)
|
||||
chunk.deviceTickMs = 77_000 + index * 50L
|
||||
writer.write(chunk)
|
||||
}
|
||||
// Two chunks were lost on the way (their ticks are skipped): a gap, not a splice.
|
||||
val late = chunk(MicStream.LEFT, 2650, 99)
|
||||
late.deviceTickMs = 77_000 + 32 * 50L
|
||||
writer.write(late)
|
||||
writer.finish()
|
||||
val times = drain(MicRecordingReader(file.input())).map { it.timeMs }
|
||||
assertEquals((0 until 30).map { it * 50L } + 1600L, times)
|
||||
assertNull(MicRecordingReader(file.input()).configJson)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aSourceWithoutAClockIsPlacedBySampleCount() {
|
||||
val file = MemoryFile()
|
||||
val writer = recorder(file)
|
||||
for (index in 0 until 10) {
|
||||
// Arrivals jitter around the capture cadence; the stream's samples are contiguous.
|
||||
val jitter = if (index % 3 == 1) 30L else 0L
|
||||
writer.write(chunk(MicStream.MONO, 400 + index * 50L + jitter, index))
|
||||
}
|
||||
// The decoder reported one packet missing before this chunk.
|
||||
val afterLoss = chunk(MicStream.MONO, 400 + 11 * 50L, 11)
|
||||
afterLoss.lostFramesBefore = 800
|
||||
writer.write(afterLoss)
|
||||
// A capture that resumes much later is placed by its arrival, not spliced on.
|
||||
writer.write(chunk(MicStream.MONO, 60_000, 12))
|
||||
writer.finish()
|
||||
val chunks = drain(MicRecordingReader(file.input()))
|
||||
assertEquals((0 until 10).map { it * 50L } + 550L + 59_600L, chunks.map { it.timeMs })
|
||||
assertEquals(800, chunks[10].lostFramesBefore)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aForeignFilePlaysItsFirstTrackAsTheMicrophone() {
|
||||
val file = MemoryFile()
|
||||
val mka = MkaWriter(file, null, 0, emptyList())
|
||||
val track = mka.addAudioTrack("Stereo", 16000, 2, false)
|
||||
// One long block of stereo frames [100, 300].
|
||||
val samples = ShortArray(2000 * 2) { if (it % 2 == 0) 100 else 300 }
|
||||
mka.writeAudio(track, 0, AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size), 0, samples.size * 2)
|
||||
mka.finish()
|
||||
val reader = MicRecordingReader(file.input())
|
||||
assertEquals("mic 2ch", reader.description)
|
||||
val chunks = drain(reader)
|
||||
assertEquals(listOf(800, 800, 400), chunks.map { it.frames })
|
||||
assertEquals(listOf(0L, 50L, 100L), chunks.map { it.timeMs })
|
||||
assertTrue(chunks.all { it.stream == MicStream.MONO && it.channels == 1 && it.pcm.all { sample -> sample == 200.toShort() } })
|
||||
|
||||
val fast = MemoryFile()
|
||||
val other = MkaWriter(fast, null, 0, emptyList())
|
||||
other.writeAudio(other.addAudioTrack("mic", 48000, 1, true), 0, pcmBytes(480, 1, 1), 0, 960)
|
||||
other.finish()
|
||||
val error = assertFailsWith<IllegalArgumentException> { MicRecordingReader(fast.input()) }
|
||||
assertTrue(error.message!!.contains("48000 Hz"), error.message)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun theFeedFollowsTheLeftTempleAndFallsBackToTheRight() {
|
||||
val mixer = SingleTempleMixer()
|
||||
// Until the left temple is heard from, the right one feeds.
|
||||
assertNotNull(mixer.mix(chunk(MicStream.RIGHT, 0, 10, channels = 2)))
|
||||
val left = chunk(MicStream.LEFT, 20, 100, channels = 2)
|
||||
left.deviceTickMs = 5
|
||||
val feed = mixer.mix(left)!!
|
||||
assertEquals(MicStream.MIX, feed.stream)
|
||||
assertEquals(1, feed.channels)
|
||||
assertEquals(800, feed.frames)
|
||||
// Channels [100, 101] average to 100.
|
||||
assertTrue(feed.pcm.all { it == 100.toShort() })
|
||||
assertEquals(MicStream.LEFT, feed.clockStream)
|
||||
assertEquals(5L, feed.deviceTickMs)
|
||||
assertNull(mixer.mix(chunk(MicStream.RIGHT, 60, 10)))
|
||||
assertNull(mixer.mix(chunk(MicStream.RIGHT, 300, 10)))
|
||||
// The left temple went quiet: the right stands in until it is back.
|
||||
assertNotNull(mixer.mix(chunk(MicStream.RIGHT, 320, 10)))
|
||||
assertNotNull(mixer.mix(chunk(MicStream.LEFT, 330, 100)))
|
||||
assertNull(mixer.mix(chunk(MicStream.RIGHT, 340, 10)))
|
||||
assertNull(mixer.mix(chunk(MicStream.MONO, 350, 10)))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun decodesTheTemplesStreamFrames() {
|
||||
val frame = micStreamFrame(sequence = 0x1234, tick = 0x89ABCDEFL, channels = 2, samples = shortArrayOf(1, -2, 3, -4), angle = -45, ssr = 300, truncated = true)
|
||||
val decoded = MicArrayProtocol.decodeFrame(frame)!!
|
||||
assertEquals(0x1234, decoded.sequence)
|
||||
assertEquals(0x89ABCDEFL, decoded.deviceTickMs)
|
||||
assertEquals(2, decoded.channels)
|
||||
assertEquals(16000, decoded.sampleRate)
|
||||
assertEquals(-45, decoded.angleDegrees)
|
||||
assertEquals(300, decoded.ssr)
|
||||
assertTrue(decoded.truncated)
|
||||
assertEquals(listOf<Short>(1, -2, 3, -4), MicArrayProtocol.pcm16(frame, decoded)!!.toList())
|
||||
assertNull(MicArrayProtocol.decodeFrame(ByteArray(205)))
|
||||
// A payload length beyond the packet is cut to what arrived.
|
||||
val short = frame.copyOf(frame.size - 4)
|
||||
assertEquals(listOf<Short>(1, -2), MicArrayProtocol.pcm16(short, MicArrayProtocol.decodeFrame(short)!!)!!.toList())
|
||||
assertEquals(listOf(0x4d, 0x43, 1, 4), MicArrayProtocol.control(MicArrayProtocol.OP_RENEW).map { it.toInt() })
|
||||
assertTrue(MicArrayProtocol.isConfigure(CONFIGURE_RECORD))
|
||||
assertFalse(MicArrayProtocol.isConfigure(MicArrayProtocol.control(MicArrayProtocol.OP_STOP)))
|
||||
}
|
||||
}
|
||||
|
||||
/** A mic_control CONFIGURE record: codec front end, both mics, raw PCM16 at 16 kHz, hardware armed. */
|
||||
internal val CONFIGURE_RECORD = byteArrayOf(0x4d, 0x43, 1, 1, 0, 3, 1, 0, 160.toByte(), 0, 0, 0, 2)
|
||||
|
||||
/** An 'SM' stream frame as a temple sends it. */
|
||||
internal fun micStreamFrame(
|
||||
sequence: Int,
|
||||
tick: Long,
|
||||
channels: Int,
|
||||
samples: ShortArray,
|
||||
angle: Int = 0,
|
||||
ssr: Int = 0,
|
||||
truncated: Boolean = false,
|
||||
): ByteArray {
|
||||
val payload = AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size)
|
||||
val header = ByteArray(MicArrayProtocol.HEADER_BYTES)
|
||||
fun le16(at: Int, value: Int) {
|
||||
header[at] = value.toByte()
|
||||
header[at + 1] = (value shr 8).toByte()
|
||||
}
|
||||
header[0] = 'S'.code.toByte()
|
||||
header[1] = 'M'.code.toByte()
|
||||
header[2] = 1
|
||||
header[3] = (if (truncated) 0x82 else 0x02).toByte()
|
||||
le16(4, sequence)
|
||||
le16(6, (tick and 0xffff).toInt())
|
||||
le16(8, ((tick shr 16) and 0xffff).toInt())
|
||||
header[10] = channels.toByte()
|
||||
le16(11, 160)
|
||||
header[13] = 0
|
||||
header[14] = 1
|
||||
le16(15, angle)
|
||||
le16(17, ssr)
|
||||
le16(19, payload.size)
|
||||
return header + payload
|
||||
}
|
||||
|
||||
class VoiceCaptureSourcesTest {
|
||||
private class Events : FaceclawVoiceControllerListener {
|
||||
val statuses = mutableListOf<String?>()
|
||||
val transcripts = mutableListOf<Pair<String?, Boolean>>()
|
||||
val pcmChunks = mutableListOf<ByteArray>()
|
||||
val replay = mutableListOf<Pair<String?, Long>>()
|
||||
val stoppedLatch = Latch(1, testPlatform())
|
||||
override fun onStatus(status: String?) { statuses.add(status) }
|
||||
override fun onTranscript(text: String?, isFinal: Boolean) { transcripts.add(text to isFinal) }
|
||||
override fun onStopped(captureId: Int) { stoppedLatch.countDown() }
|
||||
override fun onPcm(pcm16le: ByteArray?) { pcmChunks.add(pcm16le!!) }
|
||||
override fun onFrameMeta(angleDegrees: Int, ssr: Int) {}
|
||||
override fun onSpeechEnd() {}
|
||||
override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) {}
|
||||
override fun onReplay(event: String?, positionMs: Long, durationMs: Long) { replay.add(event to positionMs) }
|
||||
}
|
||||
|
||||
private class FakeLink : MicArrayLink {
|
||||
var listener: FaceclawAudioPacketListener? = null
|
||||
val controls = mutableListOf<String>()
|
||||
override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean { this.listener = listener; return true }
|
||||
override fun stopForwarding() { listener = null }
|
||||
override fun isForwarding(): Boolean = listener != null
|
||||
override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean {
|
||||
controls.add(label + ":" + record[3] + (if (leftTemple) "L" else "") + (if (rightTemple) "R" else ""))
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
private class FakeHost(val link: FakeLink? = null, val recording: MemoryFile? = null) : VoiceCaptureHost {
|
||||
override val dispatcher = CallbackDispatcher { it() }
|
||||
override fun onSessionStarting() {}
|
||||
override fun isG2SessionReady(): Boolean = link != null
|
||||
override fun isG2AudioCaptureActive(): Boolean = link?.isForwarding() == true
|
||||
override fun g2AudioSource(): PacketAudioSource? = null
|
||||
override fun createPacketDecoder(): AudioPacketDecoder = error("no stock stream in these tests")
|
||||
override fun openPhoneMic(): PcmAudioSource? = null
|
||||
override fun hasTranscriberModel(kind: VoiceModelKind): Boolean = true
|
||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber = object : OfflineTranscriber {
|
||||
override fun recognize(samples: FloatArray, count: Int): String = "x"
|
||||
override fun release() {}
|
||||
}
|
||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder = error("no verification in these tests")
|
||||
override fun micArrayLink(): MicArrayLink? = link
|
||||
override fun openRecording(path: String): SeekableInput = if (path == "missing") error("no such file") else recording!!.input()
|
||||
}
|
||||
|
||||
private val logs = mutableListOf<String>()
|
||||
|
||||
private fun session(host: VoiceCaptureHost, name: String) =
|
||||
VoiceCaptureSession(host, testPlatform(), name, { logs.add("I $it") }, { logs.add("W $it") }, { m, e -> logs.add("E $m ${e.message}") })
|
||||
|
||||
private fun waitFor(timeoutMs: Long, condition: () -> Boolean): Boolean {
|
||||
val deadline = testPlatform().elapsedRealtimeMs() + timeoutMs
|
||||
while (!condition()) {
|
||||
if (testPlatform().elapsedRealtimeMs() > deadline) return false
|
||||
sleepMs(5)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private fun monoRecording(chunks: Int): MemoryFile {
|
||||
val file = MemoryFile()
|
||||
val writer = MicRecordingWriter(file, null, 0, testPlatform())
|
||||
for (index in 0 until chunks) writer.write(chunk(MicStream.MONO, index * 50L, 1000 + index))
|
||||
writer.finish()
|
||||
return file
|
||||
}
|
||||
|
||||
@Test
|
||||
fun micArrayCaptureArmsTheTemplesRecordsThemAndTranscribesTheFeed() {
|
||||
val link = FakeLink()
|
||||
val host = FakeHost(link)
|
||||
val events = Events()
|
||||
val session = session(host, "voice-test-array")
|
||||
val file = MemoryFile()
|
||||
val recorder = MicRecordingWriter(file, "{}", 0, testPlatform())
|
||||
session.setListener(events)
|
||||
session.setMicArray(CONFIGURE_RECORD)
|
||||
session.setRecorder(recorder)
|
||||
session.start("cloud", 1)
|
||||
assertTrue(waitFor(3000) { link.controls.isNotEmpty() })
|
||||
assertTrue(session.isCapturing())
|
||||
val listener = link.listener!!
|
||||
for (index in 0 until 5) {
|
||||
val left = ShortArray(1600) { if (it % 2 == 0) 100 else 300 }
|
||||
listener.onAudioPacket(micStreamFrame(index, 1000 + index * 50L, 2, left, angle = 20, ssr = 7), "L", 500 + index * 50L)
|
||||
listener.onAudioPacket(micStreamFrame(index, 9000 + index * 50L, 2, ShortArray(1600) { 50 }), "R", 505 + index * 50L)
|
||||
}
|
||||
// A stock-format packet from before arming took effect is not audio.
|
||||
listener.onAudioPacket(ByteArray(205), "L", 760)
|
||||
assertTrue(waitFor(3000) { events.pcmChunks.size >= 5 })
|
||||
session.stop()
|
||||
assertTrue(events.stoppedLatch.await(3000))
|
||||
assertEquals(listOf("configure:1LR"), link.controls.take(1))
|
||||
assertEquals("stop:3LR", link.controls.last())
|
||||
assertFalse(link.isForwarding())
|
||||
assertFalse(session.isCapturing())
|
||||
// The feed is the left temple's microphones averaged.
|
||||
assertEquals(5, events.pcmChunks.size)
|
||||
assertEquals(1600, events.pcmChunks[0].size)
|
||||
assertTrue(AudioSegmentation.pcm16leToFloat(events.pcmChunks[0]).all { it == 200 / 32768f })
|
||||
assertTrue(events.statuses.contains("Listening (cloud)..."))
|
||||
|
||||
recorder.finish()
|
||||
assertEquals(listOf("mix", "left temple", "right temple", "frames"), MkaReader(file.input()).tracks.map { it.name })
|
||||
val recorded = drain(MicRecordingReader(file.input()))
|
||||
assertEquals(5, recorded.count { it.stream == MicStream.LEFT })
|
||||
assertEquals(5, recorded.count { it.stream == MicStream.RIGHT })
|
||||
val left = recorded.first { it.stream == MicStream.LEFT }
|
||||
assertEquals(2, left.channels)
|
||||
assertEquals(1000L, left.deviceTickMs)
|
||||
assertEquals(20, left.angleDegrees)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun micArrayCaptureNeedsGlassesThatHaveTheChannel() {
|
||||
val events = Events()
|
||||
val session = session(FakeHost(), "voice-test-array-none")
|
||||
session.setListener(events)
|
||||
session.setMicArray(CONFIGURE_RECORD)
|
||||
session.start("cloud", 1)
|
||||
assertEquals<List<String?>>(listOf("Voice control needs an active G2 connection."), events.statuses)
|
||||
// A record that is not a CONFIGURE leaves the capture on the mixed stream.
|
||||
val mixed = Events()
|
||||
session.setListener(mixed)
|
||||
session.setMicArray(MicArrayProtocol.control(MicArrayProtocol.OP_STOP))
|
||||
session.setUsePhoneMic(true)
|
||||
session.start("cloud", 2)
|
||||
assertTrue(mixed.stoppedLatch.await(3000))
|
||||
assertEquals<List<String?>>(listOf("Could not start the phone microphone."), mixed.statuses)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun replayPlaysARecordingThroughThePipelineInRealTime() {
|
||||
val host = FakeHost(recording = monoRecording(30))
|
||||
val events = Events()
|
||||
val session = session(host, "voice-test-replay")
|
||||
session.setListener(events)
|
||||
// A replay is never recorded again.
|
||||
val copy = MemoryFile()
|
||||
val recorder = MicRecordingWriter(copy, null, 0, testPlatform())
|
||||
session.setRecorder(recorder)
|
||||
session.setReplay("recording.mka", 1000)
|
||||
val startedAt = testPlatform().elapsedRealtimeMs()
|
||||
session.start("onboard", 5)
|
||||
assertTrue(events.stoppedLatch.await(5000))
|
||||
val elapsed = testPlatform().elapsedRealtimeMs() - startedAt
|
||||
// Half a second of audio, paced as it was captured.
|
||||
assertTrue(elapsed >= 400, "replay took $elapsed ms")
|
||||
assertEquals(10, events.pcmChunks.size)
|
||||
assertEquals(1020.toShort(), AudioSegmentation.pcm16le(events.pcmChunks[0][0], events.pcmChunks[0][1]))
|
||||
assertEquals("start" to 1000L, events.replay.first())
|
||||
assertEquals("end" to 1500L, events.replay.last())
|
||||
assertEquals(VoiceCaptureSession.REPLAY_FINISHED_STATUS, events.statuses.last())
|
||||
assertEquals("x" to true, events.transcripts.last())
|
||||
assertFalse(session.isCapturing())
|
||||
recorder.finish()
|
||||
assertTrue(MkaReader(copy.input()).tracks.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun replayJumpsAndPauses() {
|
||||
val host = FakeHost(recording = monoRecording(200))
|
||||
val events = Events()
|
||||
val session = session(host, "voice-test-replay-seek")
|
||||
session.setListener(events)
|
||||
session.setReplay("recording.mka", 0)
|
||||
session.start("onboard", 6)
|
||||
assertTrue(waitFor(3000) { events.pcmChunks.size >= 2 })
|
||||
session.setReplayPaused(true)
|
||||
sleepMs(100)
|
||||
val heldAt = events.pcmChunks.size
|
||||
sleepMs(200)
|
||||
assertEquals(heldAt, events.pcmChunks.size)
|
||||
assertTrue(session.isCapturing())
|
||||
// Jump to the last quarter-second while paused, then go on.
|
||||
session.seekReplay(9750)
|
||||
session.setReplayPaused(false)
|
||||
assertTrue(events.stoppedLatch.await(5000))
|
||||
assertTrue(events.replay.contains("seek" to 9750L), events.replay.toString())
|
||||
assertEquals("end" to 10_000L, events.replay.last())
|
||||
assertEquals(heldAt + 5, events.pcmChunks.size)
|
||||
// What played before the jump ended as its own utterance.
|
||||
assertEquals(2, events.transcripts.count { it.second })
|
||||
val last = events.pcmChunks.last()
|
||||
assertEquals(1199.toShort(), AudioSegmentation.pcm16le(last[0], last[1]))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun replayOfAnUnreadableFileReportsAndStops() {
|
||||
val events = Events()
|
||||
val session = session(FakeHost(), "voice-test-replay-missing")
|
||||
session.setListener(events)
|
||||
session.setReplay("missing", 0)
|
||||
session.start("cloud", 7)
|
||||
assertTrue(events.stoppedLatch.await(3000))
|
||||
assertEquals<List<String?>>(listOf("Could not open the recording: no such file"), events.statuses)
|
||||
assertEquals(listOf<Pair<String?, Long>>("error" to 0L), events.replay)
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,8 @@ class VoiceCaptureSessionTest {
|
||||
override fun onFrameMeta(angleDegrees: Int, ssr: Int) { frameMeta.add(angleDegrees to ssr) }
|
||||
override fun onSpeechEnd() { speechEnds++ }
|
||||
override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) { verified = isWearer to similarity }
|
||||
val replay = mutableListOf<Pair<String?, Long>>()
|
||||
override fun onReplay(event: String?, positionMs: Long, durationMs: Long) { replay.add(event to positionMs) }
|
||||
}
|
||||
|
||||
/** Scripted phone mic: each entry is a chunk amplitude; after the script the read fails (-1). */
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNotNull
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
import platform.Foundation.NSTemporaryDirectory
|
||||
import platform.Foundation.NSUUID
|
||||
|
||||
class IosMicRecorderTest {
|
||||
@Test
|
||||
fun recordsPcmToAMatroskaFileThroughThePosixFileAdapters() {
|
||||
val path = NSTemporaryDirectory() + NSUUID().UUIDString + ".mka"
|
||||
try {
|
||||
val recorder = assertNotNull(IosMicRecordings.open(path, "{\"platform\":\"ios\"}"))
|
||||
// 2.5 s in 50 ms chunks: enough for whole clusters to reach the file before finish().
|
||||
for (index in 0 until 50) {
|
||||
val samples = ShortArray(800) { (index * 10 + it % 7).toShort() }
|
||||
recorder.append(AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size).data())
|
||||
}
|
||||
assertFalse(recorder.failed)
|
||||
val length = recorder.finish()
|
||||
assertTrue(length >= 2500, "recorded $length ms")
|
||||
assertEquals(fileLength(path), openFileInput(path).length)
|
||||
|
||||
val reader = MicRecordingReader(openFileInput(path))
|
||||
assertEquals("{\"platform\":\"ios\"}", reader.configJson)
|
||||
assertEquals("mic", reader.description)
|
||||
var chunks = 0
|
||||
var last: MicChunk? = null
|
||||
while (true) {
|
||||
last = reader.next() ?: break
|
||||
assertEquals(800, last.frames)
|
||||
assertEquals((chunks * 10).toShort(), last.pcm[0])
|
||||
chunks++
|
||||
}
|
||||
assertEquals(50, chunks)
|
||||
reader.close()
|
||||
} finally {
|
||||
deleteFile(path)
|
||||
}
|
||||
assertNull(IosMicRecordings.open("/nonexistent-directory/x.mka", null))
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const load = loader();
|
||||
const { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts');
|
||||
const { SpeechPauseDetector } = load('app/native/speech-pause.ts');
|
||||
const { formatReplayTime, stepReplaySeconds } = load('app/apps/transcribe/replay-time.ts');
|
||||
const { TimedTranscript } = load('app/native/transcript-format.ts');
|
||||
const { bordered, unbordered, edged } = require('./helpers/viewport.cjs');
|
||||
|
||||
@@ -128,3 +129,33 @@ test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores
|
||||
d.accept(pcm(3000));
|
||||
assert.equal(d.accept(pcm(0, 1500)), true);
|
||||
});
|
||||
|
||||
test('a replay jump leaves its time on a line of its own and starts a new paragraph', () => {
|
||||
const m = new TranscriptModel();
|
||||
m.mark('[0:30]');
|
||||
assert.equal(m.text, '[0:30]');
|
||||
m.accept({ text: 'After the jump', isFinal: false });
|
||||
assert.equal(m.text, '[0:30]\nAfter the jump');
|
||||
// What was still being spoken when the replay jumped is kept, ahead of the mark.
|
||||
m.mark('[1:15]');
|
||||
m.accept({ text: 'Later.', isFinal: true });
|
||||
m.accept({ text: 'More.', isFinal: true });
|
||||
assert.equal(m.text, '[0:30]\nAfter the jump\n[1:15]\nLater. More.');
|
||||
});
|
||||
|
||||
test('replay times read as m:ss, with hours from an hour on', () => {
|
||||
assert.equal(formatReplayTime(0), '0:00');
|
||||
assert.equal(formatReplayTime(83_999), '1:23');
|
||||
assert.equal(formatReplayTime(3_600_000), '1:00:00');
|
||||
assert.equal(formatReplayTime(83_000, true), '0:01:23');
|
||||
assert.equal(formatReplayTime(-5), '0:00');
|
||||
});
|
||||
|
||||
test('the time picker steps carry between fields and stay inside the recording', () => {
|
||||
assert.equal(stepReplaySeconds(59, 'seconds', 1, 600), 60);
|
||||
assert.equal(stepReplaySeconds(60, 'seconds', -1, 600), 59);
|
||||
assert.equal(stepReplaySeconds(30, 'minutes', 2, 600), 150);
|
||||
assert.equal(stepReplaySeconds(590, 'minutes', 1, 600), 600);
|
||||
assert.equal(stepReplaySeconds(30, 'minutes', -1, 600), 0);
|
||||
assert.equal(stepReplaySeconds(100, 'hours', 1, 7300), 3700);
|
||||
});
|
||||
@@ -164,6 +164,8 @@ function androidBridge(modules = {}) {
|
||||
setListener(listener) { this.listener = listener; },
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||
setMicArray() {}, setReplay() {}, setRecordingEnabled(enabled) { this.recordingEnabled = enabled; },
|
||||
noteRecordingEvent(json) { (this.recordingEvents ??= []).push(JSON.parse(json)); },
|
||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||
stop() { this.active = false; },
|
||||
isCapturing() { return this.active; },
|
||||
@@ -217,24 +219,23 @@ test('a shared capture restarts for a holder that needs another provider, and is
|
||||
assert.equal(bridge.holdsCapture('ptt'), true);
|
||||
});
|
||||
|
||||
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
|
||||
test('Android continuous captures record to the requested file, and report one that cannot be created', () => {
|
||||
const recordings = [];
|
||||
let fail = false, attempts = 0;
|
||||
class FakeRecorder {
|
||||
static open(name) {
|
||||
attempts++;
|
||||
if (fail) return null;
|
||||
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||
}
|
||||
append(pcm) { this.pcm.push(...pcm); }
|
||||
close() { this.closed = true; }
|
||||
}
|
||||
const { bridge, controller } = androidBridge({
|
||||
'./wav-recorder': { WavRecorder: FakeRecorder },
|
||||
'./mic-recording': { newRecordingPath: name => `/Download/${name}.mka` },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
|
||||
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
|
||||
'../util/audio-level': { pcmLevel: () => 0 },
|
||||
});
|
||||
// The controller writes the file itself, from the capture's own thread.
|
||||
controller.startRecording = (path, json) => {
|
||||
attempts++;
|
||||
if (fail) return false;
|
||||
recordings.push({ path, info: JSON.parse(json), closed: false });
|
||||
return true;
|
||||
};
|
||||
controller.stopRecording = () => { recordings.at(-1).closed = true; return 0; };
|
||||
const options = { provider: 'onboard' };
|
||||
let changes = 0; bridge.onRecordingChange(() => changes++);
|
||||
bridge.recordTo('transcribe-1');
|
||||
@@ -242,20 +243,24 @@ test('Android continuous captures record to the requested WAV, and report one th
|
||||
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
|
||||
bridge.startPushToTalk(options);
|
||||
controller.listener.onPcm([9, 9]);
|
||||
assert.equal(attempts, 0);
|
||||
assert.deepEqual([attempts, controller.recordingEnabled], [0, false]);
|
||||
bridge.startContinuousCapture(options);
|
||||
controller.listener.onPcm([1, 2]);
|
||||
controller.listener.onPcm([3, 4]);
|
||||
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
|
||||
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
|
||||
assert.deepEqual(recordings.map(r => [r.path, r.info.recording, r.closed]), [['/Download/transcribe-1.mka', 'transcribe-1', false]]);
|
||||
assert.deepEqual([attempts, controller.recordingEnabled], [1, true]);
|
||||
// The file outlives a capture: one restarted after the glasses reconnect goes on in it,
|
||||
// with the restart noted, and nothing is recorded while no capture holds the mic.
|
||||
bridge.handleSessionEnded();
|
||||
assert.equal(controller.recordingEnabled, false);
|
||||
bridge.resumeCapture(options);
|
||||
controller.listener.onPcm([5]);
|
||||
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
|
||||
assert.deepEqual([attempts, recordings[0].closed, controller.recordingEnabled], [1, false, true]);
|
||||
assert.deepEqual(controller.recordingEvents.map(event => [event.event, event.recording]), [['capture', 'transcribe-1']]);
|
||||
bridge.recordTo('');
|
||||
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
|
||||
controller.listener.onPcm([6]);
|
||||
assert.equal(recordings[0].pcm.length, 5);
|
||||
assert.equal(attempts, 1);
|
||||
|
||||
// One attempt per recording, not one per packet; another recording is a fresh attempt.
|
||||
fail = true; attempts = 0;
|
||||
@@ -269,5 +274,5 @@ test('Android continuous captures record to the requested WAV, and report one th
|
||||
bridge.recordTo('');
|
||||
bridge.recordTo('transcribe-2');
|
||||
controller.listener.onPcm([10]);
|
||||
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
|
||||
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).path], [true, false, '/Download/transcribe-2.mka']);
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
/** The Android voice bridge over a fake native controller that records what it is told. */
|
||||
function harness({ recordingOk = true, downloads = '/downloads' } = {}) {
|
||||
const controllers = [];
|
||||
class FakeController {
|
||||
constructor() {
|
||||
this.calls = []; this.events = []; this.capturing = false; this.recording = null;
|
||||
this.micArray = null; this.replay = null; this.recordingEnabled = null;
|
||||
controllers.push(this);
|
||||
// Setters the tests don't look at are only logged.
|
||||
return new Proxy(this, { get: (target, name) => name in target ? target[name] : (...args) => { target.calls.push([name, ...args]); } });
|
||||
}
|
||||
setListener(listener) { this.listener = listener; }
|
||||
setMicArray(record) { this.micArray = record; }
|
||||
setReplay(path, startMs) { this.replay = path === null ? null : { path, startMs }; }
|
||||
start(mode) { this.calls.push(['start', mode]); this.capturing = true; }
|
||||
stop() { this.calls.push(['stop']); this.capturing = false; }
|
||||
isCapturing() { return this.capturing; }
|
||||
startRecording(path, json) { if (recordingOk) this.recording = { path, info: JSON.parse(json) }; return recordingOk; }
|
||||
setRecordingEnabled(enabled) { this.recordingEnabled = enabled; }
|
||||
noteRecordingEvent(json) { this.events.push(JSON.parse(json)); }
|
||||
stopRecording() { this.calls.push(['stopRecording']); this.recording = null; return 0; }
|
||||
seekReplay(ms) { this.calls.push(['seekReplay', ms]); }
|
||||
setReplayPaused(paused) { this.calls.push(['setReplayPaused', paused]); }
|
||||
}
|
||||
const module = loader({
|
||||
global: { isAndroid: true },
|
||||
com: { faceclaw: { app: { FaceclawVoiceController: FakeController, FaceclawVoiceControllerListener: function (impl) { return impl; } } } },
|
||||
}, {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./cloud-stt': { toJavaBytes: bytes => Array.from(bytes) },
|
||||
'./cloud-stt-provider': {
|
||||
createCloudSttClient: () => null,
|
||||
transcriptionKey: options => `${options.provider}:${options.language ?? ''}`,
|
||||
},
|
||||
'./mic-recording': { newRecordingPath: name => downloads === null ? null : `${downloads}/${name}.mka` },
|
||||
'../util/array-util': { toUint8Array: value => value },
|
||||
})('app/native/voice-control.ts');
|
||||
const options = (extra = {}) => ({
|
||||
communicator: {}, provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '', ...extra,
|
||||
});
|
||||
return { ...module, controllers, options, pcm: controller => controller.listener.onPcm(new Uint8Array(1600)) };
|
||||
}
|
||||
|
||||
const CONFIGURE_2 = Uint8Array.from([0x4d, 0x43, 1, 1, 1, 1, 1, 0, 160, 0, 0, 0, 2]);
|
||||
const CONFIGURE_4 = Uint8Array.from([0x4d, 0x43, 1, 1, 0, 3, 1, 0, 160, 0, 0, 0, 2]);
|
||||
|
||||
test('a capture of the temples\' microphones arms them, and another microphone count restarts the stream', () => {
|
||||
const h = harness();
|
||||
const bridge = h.voiceControlBridge;
|
||||
bridge.startContinuousCapture(h.options({ micArray: { configure: CONFIGURE_2 } }));
|
||||
const controller = h.controllers[0];
|
||||
assert.deepEqual(controller.micArray, Array.from(CONFIGURE_2));
|
||||
assert.equal(controller.replay, null);
|
||||
assert.deepEqual(controller.calls.filter(call => call[0] === 'start'), [['start', 'onboard']]);
|
||||
// Push-to-talk asks for the same thing while the window is open: it shares the stream.
|
||||
bridge.startPushToTalk(h.options({ micArray: { configure: CONFIGURE_2 } }));
|
||||
assert.equal(controller.calls.filter(call => call[0] === 'start').length, 1);
|
||||
bridge.stopPushToTalk();
|
||||
// The Microphones setting changed.
|
||||
bridge.retargetContinuousCapture(h.options({ micArray: { configure: CONFIGURE_4 } }));
|
||||
assert.deepEqual(controller.micArray, Array.from(CONFIGURE_4));
|
||||
assert.deepEqual(controller.calls.filter(call => call[0] === 'start' || call[0] === 'stop').map(call => call[0]), ['start', 'stop', 'start']);
|
||||
// Back to the mixed stream; the phone's microphone never gets an array.
|
||||
bridge.retargetContinuousCapture(h.options());
|
||||
assert.equal(controller.micArray, null);
|
||||
bridge.retargetContinuousCapture(h.options({ communicator: null, usePhoneMic: true, micArray: { configure: CONFIGURE_4 } }));
|
||||
assert.equal(controller.micArray, null);
|
||||
});
|
||||
|
||||
test('a recording opens with the first audio, describes its capture, and follows the capture across restarts', () => {
|
||||
const h = harness();
|
||||
const bridge = h.voiceControlBridge;
|
||||
const changes = [];
|
||||
bridge.onRecordingChange(() => changes.push(bridge.recording));
|
||||
bridge.recordTo('transcribe-1');
|
||||
assert.equal(bridge.recording, true);
|
||||
bridge.startContinuousCapture(h.options({ micArray: { configure: CONFIGURE_4 }, noiseSuppression: true, captureInfo: { mics: 4, glasses: 'G2' } }));
|
||||
const controller = h.controllers[0];
|
||||
// Nothing on disk until audio arrives: a capture that never starts leaves no file.
|
||||
assert.equal(controller.recording, null);
|
||||
assert.equal(controller.recordingEnabled, true);
|
||||
h.pcm(controller);
|
||||
assert.equal(controller.recording.path, '/downloads/transcribe-1.mka');
|
||||
const info = controller.recording.info;
|
||||
assert.equal(info.recording, 'transcribe-1');
|
||||
assert.equal(info.source, 'glasses');
|
||||
assert.equal(info.micArray, true);
|
||||
assert.equal(info.mics, 4);
|
||||
assert.equal(info.glasses, 'G2');
|
||||
assert.deepEqual(info.transcription, { provider: 'onboard', language: '' });
|
||||
assert.deepEqual(info.processing, { noiseSuppression: true, beamFilter: null });
|
||||
h.pcm(controller);
|
||||
assert.equal(controller.calls.filter(call => call[0] === 'stopRecording').length, 0);
|
||||
|
||||
// The capture restarts with other microphones: the same file goes on, with the change noted in it.
|
||||
bridge.retargetContinuousCapture(h.options({ captureInfo: { mics: 1, glasses: 'G2' } }));
|
||||
assert.equal(controller.events.length, 1);
|
||||
assert.equal(controller.events[0].event, 'capture');
|
||||
assert.equal(controller.events[0].mics, 1);
|
||||
assert.equal(controller.events[0].micArray, false);
|
||||
assert.equal(controller.recording.path, '/downloads/transcribe-1.mka');
|
||||
|
||||
// Save Recording was turned off.
|
||||
bridge.recordTo('');
|
||||
assert.equal(bridge.recording, false);
|
||||
assert.equal(controller.recording, null);
|
||||
assert.deepEqual(changes, [true, false]);
|
||||
});
|
||||
|
||||
test('push-to-talk on a capture of its own is not recorded', () => {
|
||||
const h = harness();
|
||||
const bridge = h.voiceControlBridge;
|
||||
bridge.recordTo('transcribe-1');
|
||||
bridge.startPushToTalk(h.options());
|
||||
const controller = h.controllers[0];
|
||||
assert.equal(controller.recordingEnabled, false);
|
||||
h.pcm(controller);
|
||||
assert.equal(controller.recording, null);
|
||||
// The Transcribe window joins the running stream: from here it is recorded.
|
||||
bridge.startContinuousCapture(h.options());
|
||||
assert.equal(controller.recordingEnabled, true);
|
||||
h.pcm(controller);
|
||||
assert.equal(controller.recording.path, '/downloads/transcribe-1.mka');
|
||||
bridge.stopContinuousCapture();
|
||||
assert.equal(controller.recordingEnabled, false);
|
||||
});
|
||||
|
||||
test('a recording that cannot be created is reported once', () => {
|
||||
for (const h of [harness({ recordingOk: false }), harness({ downloads: null })]) {
|
||||
const bridge = h.voiceControlBridge;
|
||||
bridge.recordTo('transcribe-1');
|
||||
bridge.startContinuousCapture(h.options());
|
||||
h.pcm(h.controllers[0]);
|
||||
assert.equal(bridge.recording, false);
|
||||
assert.equal(bridge.recordingFailed, true);
|
||||
}
|
||||
});
|
||||
|
||||
test('a replay runs on its own controller, follows its events, jumps while playing and restarts after the end', () => {
|
||||
const h = harness();
|
||||
const replay = new h.VoiceReplay('/downloads/talk.mka', () => h.options({ communicator: {}, micArray: { configure: CONFIGURE_4 } }));
|
||||
const seen = [];
|
||||
replay.onChange(jumpedToMs => seen.push([replay.state, replay.positionMs, jumpedToMs]));
|
||||
assert.equal(h.controllers.length, 0);
|
||||
replay.start();
|
||||
// The microphone's bridge has not been touched.
|
||||
assert.equal(h.controllers.length, 1);
|
||||
const controller = h.controllers[0];
|
||||
assert.deepEqual(controller.replay, { path: '/downloads/talk.mka', startMs: 0 });
|
||||
assert.equal(controller.micArray, null);
|
||||
assert.equal(replay.state, 'opening');
|
||||
const statuses = [];
|
||||
replay.events.onStatus(state => statuses.push(state.status));
|
||||
assert.equal(statuses.at(-1), 'Opening the recording...');
|
||||
|
||||
controller.listener.onReplay('start', 0, 60_000);
|
||||
assert.equal(replay.state, 'playing');
|
||||
assert.equal(replay.durationMs, 60_000);
|
||||
controller.listener.onReplay('position', 1000, 60_000);
|
||||
assert.equal(replay.positionMs, 1000);
|
||||
// A plain start and the ticking position are not jumps.
|
||||
assert.deepEqual(seen.map(entry => entry[2]), [undefined, undefined, undefined]);
|
||||
|
||||
replay.setPaused(true);
|
||||
assert.equal(replay.state, 'paused');
|
||||
replay.setPaused(false);
|
||||
replay.jumpTo(90_000); // past the end: held to the recording's length
|
||||
assert.deepEqual(controller.calls.filter(call => call[0] === 'seekReplay' || call[0] === 'setReplayPaused'),
|
||||
[['setReplayPaused', true], ['setReplayPaused', false], ['seekReplay', 60_000]]);
|
||||
replay.jumpTo(30_000);
|
||||
controller.listener.onReplay('seek', 30_000, 60_000);
|
||||
assert.deepEqual(seen.at(-1), ['playing', 30_000, 30_000]);
|
||||
|
||||
controller.listener.onReplay('end', 60_000, 60_000);
|
||||
controller.capturing = false;
|
||||
assert.equal(replay.state, 'ended');
|
||||
assert.equal(replay.positionMs, 60_000);
|
||||
replay.setPaused(true); // nothing to pause
|
||||
assert.equal(replay.state, 'ended');
|
||||
|
||||
// A jump once it has ended plays again from there.
|
||||
replay.jumpTo(45_000);
|
||||
assert.deepEqual(controller.replay, { path: '/downloads/talk.mka', startMs: 45_000 });
|
||||
assert.equal(replay.state, 'opening');
|
||||
controller.listener.onReplay('start', 45_000, 60_000);
|
||||
assert.deepEqual(seen.at(-1), ['playing', 45_000, 45_000]);
|
||||
|
||||
replay.close();
|
||||
assert.equal(controller.calls.at(-1)[0], 'stop');
|
||||
replay.start();
|
||||
assert.equal(controller.calls.at(-1)[0], 'stop');
|
||||
});
|
||||
|
||||
test('a recording that cannot be opened fails the replay', () => {
|
||||
const h = harness();
|
||||
const replay = new h.VoiceReplay('/downloads/missing.mka', () => h.options());
|
||||
replay.start();
|
||||
const controller = h.controllers[0];
|
||||
controller.listener.onStatus('Could not open the recording: no such file');
|
||||
controller.listener.onReplay('error', 0, 0);
|
||||
assert.equal(replay.state, 'failed');
|
||||
const statuses = [];
|
||||
replay.events.onStatus(state => statuses.push(state.status));
|
||||
assert.deepEqual(statuses, ['Could not open the recording: no such file']);
|
||||
});
|
||||
@@ -1,19 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
test('the WAV header describes 16 kHz mono PCM16 and its data size', () => {
|
||||
const { wavHeader } = loader({ DataView }, {})('app/native/wav-header.ts');
|
||||
const header = Buffer.from(wavHeader(32000));
|
||||
assert.equal(header.length, 44);
|
||||
assert.equal(header.toString('ascii', 0, 4), 'RIFF');
|
||||
assert.equal(header.readUInt32LE(4), 36 + 32000);
|
||||
assert.equal(header.toString('ascii', 8, 16), 'WAVEfmt ');
|
||||
assert.equal(header.readUInt16LE(20), 1);
|
||||
assert.equal(header.readUInt16LE(22), 1);
|
||||
assert.equal(header.readUInt32LE(24), 16000);
|
||||
assert.equal(header.readUInt32LE(28), 32000);
|
||||
assert.equal(header.readUInt16LE(34), 16);
|
||||
assert.equal(header.toString('ascii', 36, 40), 'data');
|
||||
assert.equal(header.readUInt32LE(40), 32000);
|
||||
});
|
||||
Reference in new issue
Block a user