forked from Blendman974/faceclaw
Multi-mic support and replay mode for Transcribe app
This commit is contained in:
1 parent
c162c8e633
commit
9372b2742c
45 files changed
+3926
-288
No files matched your search
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { isMicRecordingFile } from "../../native/mic-recording";
|
||||
import { isPlayableVideoFile } from "../../native/video-player";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
openFontWindow: (title: string, path: string) => void;
|
||||
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||
openVideoWindow: (title: string, path: string) => void;
|
||||
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
|
||||
openRecordingInTranscribe: (title: string, path: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) =>
|
||||
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isMicRecordingFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
label: "Open in Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openRecordingInTranscribe(entry.name, entry.path);
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isPlayableVideoFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
FILES_WINDOW_ID,
|
||||
} from "./files-app";
|
||||
import { createVideoPlayerWindow } from "./video-window";
|
||||
import { openRecordingInTranscribe } from "../transcribe";
|
||||
|
||||
// Document windows are closeable and non-singleton; the serial keeps their
|
||||
// windowIds unique.
|
||||
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
|
||||
},
|
||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||
openRecordingInTranscribe: (title, path) => {
|
||||
openRecordingInTranscribe(ctx, path, title);
|
||||
},
|
||||
}),
|
||||
),
|
||||
// A document arriving via Android's Share intent opens as its own window.
|
||||
|
||||
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration a capture of the temples' own microphones arms them
|
||||
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
|
||||
* of each. The front end and direction detection follow Microphone setup
|
||||
* (the Microphones app), except that both of a temple's microphones only
|
||||
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
|
||||
* what the firmware streams and transcription takes.
|
||||
*/
|
||||
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
|
||||
const base = loadMicConfig();
|
||||
return {
|
||||
...base,
|
||||
source: mics === 4 ? "codec" : base.source,
|
||||
channelMask: mics === 4 ? 0b11 : 0b01,
|
||||
codec: "raw",
|
||||
format: "pcm16",
|
||||
sampleRateHz: 16_000,
|
||||
armHardware: true,
|
||||
};
|
||||
}
|
||||
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
|
||||
@@ -1,5 +1,32 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
|
||||
import { type AppContext, type AppDefinition } from "../app-definition";
|
||||
import {
|
||||
createTranscribeAppWindow,
|
||||
createTranscribeReplayWindow,
|
||||
TRANSCRIBE_SURFACE_ID,
|
||||
TRANSCRIBE_WINDOW_ID,
|
||||
} from "./transcribe-app";
|
||||
|
||||
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
|
||||
let nextReplaySerial = 1;
|
||||
|
||||
/**
|
||||
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
|
||||
* own. Returns false where recordings can't be replayed.
|
||||
*/
|
||||
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
|
||||
if (!ctx.actions.openVoiceReplay) return false;
|
||||
const windowId = `transcribe:replay:${nextReplaySerial++}`;
|
||||
void ctx
|
||||
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
|
||||
const window = createTranscribeReplayWindow(windowId, path, name, options);
|
||||
if (!window) throw new Error("recordings can't be replayed here");
|
||||
return window;
|
||||
})
|
||||
.catch((error) => {
|
||||
ctx.appendLog(`transcribe replay window failed: ${error}`);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const transcribeApp: AppDefinition = {
|
||||
appId: "transcribe",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
|
||||
export function formatReplayTime(ms: number, withHours = false): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const seconds = String(total % 60).padStart(2, "0");
|
||||
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
|
||||
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
|
||||
}
|
||||
|
||||
/** The time picker's fields, in display and click order. */
|
||||
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
|
||||
export type TimeField = (typeof TIME_FIELDS)[number];
|
||||
|
||||
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
|
||||
|
||||
/**
|
||||
* Step a position (whole seconds) by `delta` of a field's unit, carrying
|
||||
* into the neighboring fields, within [0, maxSeconds].
|
||||
*/
|
||||
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
|
||||
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
|
||||
import { CONTEXT_MENU_DIM } from "../../ui/menu";
|
||||
import { centeredTextY } from "../../ui/metrics";
|
||||
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
|
||||
|
||||
const PADDING = 12;
|
||||
const FIELD_GAP = 6;
|
||||
const FIELD_PAD_X = 6;
|
||||
/** Fill and outline of the active field (the Timers and date pickers' field style). */
|
||||
const ACTIVE_FILL = 35;
|
||||
const ACTIVE_STROKE = 130;
|
||||
|
||||
export type TimePickerOptions = {
|
||||
title: string;
|
||||
/** Where the picker starts, and the furthest it goes, in ms. */
|
||||
initialMs: number;
|
||||
maxMs: number;
|
||||
onPick: (positionMs: number) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* A modal position picker for a recording: hours (when the recording has
|
||||
* any), minutes and seconds, over the dimmed content beneath. Input follows
|
||||
* the date picker's: scroll adjusts the active field (up is later, carrying
|
||||
* into the fields beside it), click moves to the next field and on the last
|
||||
* one picks, double-click steps back a field and on the first one cancels.
|
||||
* A watch's swipes move between fields and adjust.
|
||||
*/
|
||||
export class TimePickerLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
readonly dimUnderneath = CONTEXT_MENU_DIM;
|
||||
private readonly fields: readonly TimeField[];
|
||||
private readonly maxSeconds: number;
|
||||
private seconds: number;
|
||||
private field = 0;
|
||||
|
||||
constructor(private readonly options: TimePickerOptions) {
|
||||
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
|
||||
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
|
||||
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "swipe-up":
|
||||
this.adjust(1);
|
||||
return;
|
||||
case "scroll-down":
|
||||
case "swipe-down":
|
||||
this.adjust(-1);
|
||||
return;
|
||||
case "swipe-right":
|
||||
this.field = Math.min(this.fields.length - 1, this.field + 1);
|
||||
return;
|
||||
case "swipe-left":
|
||||
this.field = Math.max(0, this.field - 1);
|
||||
return;
|
||||
case "click":
|
||||
if (this.field < this.fields.length - 1) {
|
||||
this.field++;
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
this.options.onPick(this.seconds * 1000);
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
if (this.field > 0) this.field--;
|
||||
else ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private adjust(delta: number): void {
|
||||
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
|
||||
}
|
||||
|
||||
private hints(field: number): string {
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, "adjust"],
|
||||
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
|
||||
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
|
||||
]);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const image = paintBelow();
|
||||
const small = getDefaultSmallFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const focused = ctx.stack.isFocused();
|
||||
const base = ctx.stack.getBaseSize();
|
||||
|
||||
const values = [
|
||||
String(Math.floor(this.seconds / 3600)),
|
||||
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
|
||||
String(this.seconds % 60).padStart(2, "0"),
|
||||
].slice(TIME_FIELDS.length - this.fields.length);
|
||||
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
|
||||
const fieldWidths = this.fields.map(
|
||||
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
|
||||
);
|
||||
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
|
||||
const boxHeight = medium.lineHeight + 6;
|
||||
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
|
||||
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
|
||||
const width = Math.min(
|
||||
base.width,
|
||||
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
|
||||
);
|
||||
const titleHeight = small.lineHeight + 12;
|
||||
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
|
||||
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
|
||||
const x = Math.max(0, Math.floor((base.width - width) / 2));
|
||||
const y = Math.max(0, Math.floor((base.height - height) / 2));
|
||||
|
||||
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
|
||||
image.fillRoundedRect(x, y, width, height, 1);
|
||||
image.drawRoundedRect(x, y, width, height, 72);
|
||||
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
|
||||
|
||||
const top = y + 8 + titleHeight;
|
||||
let fieldX = x + PADDING;
|
||||
for (let index = 0; index < this.fields.length; index++) {
|
||||
const fieldWidth = fieldWidths[index]!;
|
||||
const active = index === this.field;
|
||||
if (active) {
|
||||
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
|
||||
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
|
||||
}
|
||||
const value = values[index]!;
|
||||
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
|
||||
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
|
||||
const caption = this.fields[index]!;
|
||||
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
|
||||
top + boxHeight + 2, caption, active ? 190 : 110);
|
||||
fieldX += fieldWidth + FIELD_GAP;
|
||||
}
|
||||
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
|
||||
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
import { withRecordingIcon } from "../../ui/listening-header";
|
||||
import { TimePickerLayer } from "./time-picker";
|
||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
|
||||
}
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
// One timestamp per session names both the audio recording and a saved transcript.
|
||||
const stamp = transcriptTimestamp();
|
||||
const layer = new TranscribeLayer(stamp);
|
||||
// At launch, and again when the Microphones setting changes: a running
|
||||
// capture moves over to the microphones now asked for.
|
||||
const startCapture = () =>
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
},
|
||||
// The controller starts and stops the recording as the setting changes.
|
||||
toggleSettingMenuItem(saveRecordingSetting),
|
||||
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
app.requestRender();
|
||||
});
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
startCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
return app;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe in replay mode: a window of its own that transcribes a
|
||||
* recording (a .mka file from Save Recording) in place of the microphones,
|
||||
* in real time and through the same pipeline. It neither uses the mic nor
|
||||
* shows the mic tray icon, so it runs beside a live Transcribe window. The
|
||||
* header shows the position in the recording; a tap pauses and resumes; the
|
||||
* menu jumps to a time.
|
||||
*/
|
||||
export function createTranscribeReplayWindow(
|
||||
windowId: string,
|
||||
path: string,
|
||||
name: string,
|
||||
options: InProcessAppOptions,
|
||||
): InProcessWindow | null {
|
||||
const replay = options.actions.openVoiceReplay?.(path) ?? null;
|
||||
if (!replay) return null;
|
||||
// Names a saved transcript after the recording it came from.
|
||||
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId,
|
||||
title: name,
|
||||
iconLetter: "Tr",
|
||||
icon: "mic",
|
||||
closeable: true,
|
||||
// Relaunching Transcribe after a restart would open the microphone, not this recording.
|
||||
restoresApp: false,
|
||||
// Like a playing video: something to watch, not to touch.
|
||||
keepsScreenOn: () => replay.state === "playing",
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [
|
||||
{
|
||||
label: "Jump to time",
|
||||
// The length isn't known until the recording has been opened.
|
||||
disabled: () => replay.durationMs <= 0,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(
|
||||
new TimePickerLayer({
|
||||
title: "Jump to time",
|
||||
initialMs: replay.positionMs,
|
||||
maxMs: replay.durationMs,
|
||||
onPick: (positionMs) => {
|
||||
replay.jumpTo(positionMs);
|
||||
app.requestRender();
|
||||
},
|
||||
}),
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: replay.state === "paused" ? "Resume" : "Pause",
|
||||
disabled: () => replay.state !== "playing" && replay.state !== "paused",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.setPaused(replay.state === "playing");
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Restart from 0:00",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.jumpTo(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
},
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
replay.close();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// Started first, so the layer picks its opening status up as it subscribes.
|
||||
replay.start();
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
|
||||
import { type MicCount } from "../../ui/layers";
|
||||
import { micControlSupported } from "../microphones/mic-control";
|
||||
|
||||
export type TranscribeMics = "1" | "2" | "4";
|
||||
|
||||
const MICS_LABELS: Record<TranscribeMics, string> = {
|
||||
"1": "1 (mixed)",
|
||||
"2": "2 (one per temple)",
|
||||
"4": "4 (two per temple)",
|
||||
};
|
||||
|
||||
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
|
||||
export function separateMicsSupported(): boolean {
|
||||
return glassesTypeSetting.get() === "g2" && micControlSupported();
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones Transcribe captures. With more than
|
||||
* one, the temples stream their own microphones (each temple on its own
|
||||
* clock) instead of the glasses' mixed stream; a recording keeps every one,
|
||||
* and what is transcribed is a mix made on the phone.
|
||||
*/
|
||||
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
|
||||
id: "transcribe-mics",
|
||||
label: "Microphones",
|
||||
storageKey: "transcribe.mics",
|
||||
defaultValue: "1",
|
||||
values: ["1", "2", "4"],
|
||||
formatValue: (value) => MICS_LABELS[value],
|
||||
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
|
||||
description:
|
||||
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
|
||||
});
|
||||
|
||||
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
|
||||
export function transcribeMicCount(): MicCount {
|
||||
if (!separateMicsSupported()) return 1;
|
||||
return Number(transcribeMicsSetting.get()) as MicCount;
|
||||
}
|
||||
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { formatReplayTime } from "./replay-time";
|
||||
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*
|
||||
* Given a replay, it transcribes that recording instead of the microphone:
|
||||
* the header shows the position in the recording, a tap pauses and resumes,
|
||||
* and each jump leaves its time as a line in the transcript.
|
||||
*/
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeReplay: (() => void) | null = null;
|
||||
private readonly source: VoiceEventSource;
|
||||
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
/**
|
||||
* `stamp` names the saved transcript, matching the session's audio
|
||||
* recording. `replay` is the recording to transcribe in place of the
|
||||
* microphone; `replayName` is what to call it.
|
||||
*/
|
||||
constructor(
|
||||
private readonly stamp: string = transcriptTimestamp(),
|
||||
private readonly replay: VoiceReplay | null = null,
|
||||
private readonly replayName = "",
|
||||
) {
|
||||
this.source = replay ? replay.events : voiceControlBridge;
|
||||
if (replay) this.status = "Opening the recording...";
|
||||
}
|
||||
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.meter.start(requestRender, isVisible, this.source);
|
||||
this.unsubscribeTranscript = this.source.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
|
||||
// The current status arrives while subscribing, before the window has a
|
||||
// surface to render to; the window paints it when it is first shown.
|
||||
let subscribing = true;
|
||||
this.unsubscribeStatus = this.source.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
if (!subscribing) requestRender();
|
||||
});
|
||||
subscribing = false;
|
||||
const replay = this.replay;
|
||||
if (replay) {
|
||||
let shownSecond = -1;
|
||||
let shownState = replay.state;
|
||||
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
|
||||
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
|
||||
// The clock ticks in whole seconds; repaint when what it shows changed.
|
||||
const second = Math.floor(replay.positionMs / 1000);
|
||||
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
|
||||
shownSecond = second;
|
||||
shownState = replay.state;
|
||||
if (isVisible()) requestRender();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
|
||||
private replayStatus(replay: VoiceReplay): string {
|
||||
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
|
||||
const withHours = replay.durationMs >= 3_600_000;
|
||||
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
|
||||
if (replay.state === "paused") return `${clock} · Paused`;
|
||||
if (replay.state === "ended") return `${clock} · Finished`;
|
||||
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
|
||||
return this.listening ? clock : `${clock} · ${this.status}`;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
|
||||
const viewport = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(viewport.width, viewport.height, 0);
|
||||
|
||||
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const replay = this.replay;
|
||||
const status = replay
|
||||
? this.replayStatus(replay)
|
||||
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const playing = !replay || replay.state === "playing";
|
||||
const bodyTop = paintListeningHeader(
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
|
||||
voiceControlBridge.recording,
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
|
||||
this.source.recording,
|
||||
);
|
||||
const step = lineStep(font) + 2;
|
||||
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
|
||||
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
|
||||
|
||||
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
|
||||
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click" && this.replay) {
|
||||
this.replay.setPaused(this.replay.state === "playing");
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribeReplay?.();
|
||||
this.unsubscribeReplay = null;
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -36,6 +36,18 @@ export class TranscriptModel {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* A line of its own (where a replay jumped to), ending whatever was in
|
||||
* progress: what follows it starts a new paragraph.
|
||||
*/
|
||||
mark(label: string): void {
|
||||
const before = this.text;
|
||||
this.finalized = before ? `${before}\n${label}` : label;
|
||||
this.live = "";
|
||||
this.separator = "\n";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
|
||||
@@ -53,7 +53,7 @@ import { type GlanceEvent } from "./glance-state";
|
||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { VoiceReplay, voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceActivity } from "../ui/shell/voice-activity";
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
@@ -76,7 +76,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { sameEdges } from "../ui/viewport";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
@@ -98,6 +98,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
|
||||
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
|
||||
import { wearerVerificationOptions } from "../apps/microphones/speakers";
|
||||
import { micSession } from "../apps/microphones/mic-session";
|
||||
import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control";
|
||||
import { encodeMicControl } from "../apps/microphones/mic-protocol";
|
||||
import {
|
||||
chargingPreviewMessage,
|
||||
glassesDisplayAsleep,
|
||||
@@ -410,6 +412,7 @@ class DashboardController {
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
|
||||
this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
|
||||
openVoiceReplay: (path: string) => this.openVoiceReplay(path),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
@@ -2253,6 +2256,30 @@ class DashboardController {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones the continuous-capture windows call
|
||||
* for (the most any asks for), where the glasses can stream them
|
||||
* separately: G2 with the microphone channel in use. Otherwise 1, the
|
||||
* mixed stream.
|
||||
*/
|
||||
private continuousMics(): MicCount {
|
||||
if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1;
|
||||
let mics: MicCount = 1;
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if ((options.mics ?? 1) > mics) mics = options.mics!;
|
||||
}
|
||||
return mics;
|
||||
}
|
||||
|
||||
/**
|
||||
* A recording as a capture of its own (see VoiceReplay), transcribed with
|
||||
* the provider and processing settings a microphone capture would get.
|
||||
* "My voice only" is left out: a recording is of whoever spoke.
|
||||
*/
|
||||
private openVoiceReplay(path: string): VoiceReplay {
|
||||
return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous capture's claim on the mic (the Microphones app and EvenHub
|
||||
* mic apps stand down while it is held) follows whether the capture really
|
||||
@@ -2278,9 +2305,26 @@ class DashboardController {
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
// Separate microphones only come from connected glasses.
|
||||
const mics = communicator ? this.continuousMics() : 1;
|
||||
const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics);
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null,
|
||||
// Stored in a recording of the capture (see Save Recording).
|
||||
captureInfo: {
|
||||
glasses: communicator ? glassesModelName() : null,
|
||||
mics,
|
||||
micConfig: micConfig && {
|
||||
source: micConfig.source,
|
||||
channelMask: micConfig.channelMask,
|
||||
codec: micConfig.codec,
|
||||
format: micConfig.format,
|
||||
sampleRateHz: micConfig.sampleRateHz,
|
||||
directionDetection: micConfig.beamform,
|
||||
},
|
||||
},
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
@@ -2721,10 +2765,11 @@ class DashboardController {
|
||||
if (!this.openAppsRestored || this.suppressOpenAppsPersist) return;
|
||||
const open: string[] = [];
|
||||
for (const window of shell.getWindows()) {
|
||||
if (window.appId === "launcher") continue;
|
||||
if (window.appId === "launcher" || window.restoresApp === false) continue;
|
||||
if (!open.includes(window.appId)) open.push(window.appId);
|
||||
}
|
||||
savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null });
|
||||
const foreground = shell.foregroundWindow();
|
||||
savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null });
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1111,8 +1111,8 @@ export class IosPreviewController {
|
||||
/**
|
||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||
* Recording is on.
|
||||
* on Android. `recordAs` names a recording (.mka) of the audio in Downloads
|
||||
* when Save Recording is on.
|
||||
*/
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
|
||||
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
|
||||
declare const FaceclawKitIosMicRecordings: any
|
||||
|
||||
/**
|
||||
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
|
||||
* Documents/Downloads while it is captured, through the shared Kotlin writer
|
||||
* that Android's captures are recorded with. The file is written about a
|
||||
* second at a time, so a crash leaves everything up to the last second.
|
||||
*/
|
||||
export class MicRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/**
|
||||
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
|
||||
* `info` describes the capture and is stored in the file.
|
||||
*/
|
||||
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.mka`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
|
||||
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
|
||||
return recorder ? new MicRecorder(recorder, path) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.recorder || !pcm.length) return
|
||||
// The handle is the buffer's start: a view into a larger buffer is copied out first.
|
||||
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
|
||||
try {
|
||||
// Copied by the recorder before this returns.
|
||||
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
this.close()
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.recorder?.finish() } catch { /* already closed */ }
|
||||
this.recorder = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
|
||||
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
|
||||
export const RECORDING_EXTENSION = ".mka";
|
||||
|
||||
export function isMicRecordingFile(name: string): boolean {
|
||||
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
|
||||
}
|
||||
|
||||
/**
|
||||
* The path for a new recording in the public Downloads directory:
|
||||
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
|
||||
* Downloads isn't available. The file itself is created by whoever records.
|
||||
*/
|
||||
export function newRecordingPath(name: string): string | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
|
||||
return String(file.getAbsolutePath());
|
||||
} catch (error) {
|
||||
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { MicRecorder } from './mic-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
|
||||
private continuous = false
|
||||
// Shown instead of the native "ended" status once a drain the bridge started completes.
|
||||
private drainStatus: string | null = null
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
private recorder: MicRecorder | null = null
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false
|
||||
private readonly recordingListeners = new Set<() => void>()
|
||||
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or ''. Only cloud captures supply PCM.
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
|
||||
private record(pcm: Uint8Array): void {
|
||||
if (!this.recording) return
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName)
|
||||
this.recorder = MicRecorder.open(this.recordingName, {
|
||||
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
|
||||
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
|
||||
})
|
||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||
}
|
||||
|
||||
+319
-27
@@ -1,9 +1,9 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { newRecordingPath } from "./mic-recording";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { WavRecorder } from "./wav-recorder";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
@@ -84,6 +84,49 @@ export type PushToTalkOptions = {
|
||||
* frame). Null/absent leaves the mic omnidirectional.
|
||||
*/
|
||||
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
|
||||
/**
|
||||
* Capture from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the glasses' mixed stream. Each temple
|
||||
* streams its microphones on its own link and clock; the phone mixes them
|
||||
* into the mono feed that is transcribed. Null/absent keeps the mixed
|
||||
* stream; ignored with usePhoneMic.
|
||||
*/
|
||||
micArray?: MicArrayCapture | null;
|
||||
/**
|
||||
* Play this recording (a .mka file from Save Recording) in real time in
|
||||
* place of any microphone. The capture needs no glasses, and ends by
|
||||
* itself when the recording does.
|
||||
*/
|
||||
replay?: { path: string; startMs: number } | null;
|
||||
/** What the caller knows of the capture, for the description stored in a recording of it. */
|
||||
captureInfo?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type MicArrayCapture = {
|
||||
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
|
||||
configure: Uint8Array;
|
||||
};
|
||||
|
||||
/** Progress of a capture that replays a recording; positions are ms into it. */
|
||||
export type VoiceReplayEvent = {
|
||||
/**
|
||||
* `start` once the file is open, `position` about once per second played,
|
||||
* `seek` when a jump took effect, `end` when the recording ran out, and
|
||||
* `error` when it could not be opened (the status says why).
|
||||
*/
|
||||
kind: "start" | "position" | "seek" | "end" | "error";
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
|
||||
export type VoiceEventSource = {
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void;
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
|
||||
onSpeechPause(listener: () => void): () => void;
|
||||
onMicLevel(listener: (level: number) => void): () => void;
|
||||
readonly recording: boolean;
|
||||
readonly recordingFailed: boolean;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
@@ -130,9 +173,13 @@ export class FaceclawVoiceControlBridge {
|
||||
private captureEndpointing = false;
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// What the running STT stream transcribes with (transcriptionKey); a holder
|
||||
// asking for something else restarts the stream rather than sharing it.
|
||||
// What the running STT stream captures from and transcribes with
|
||||
// (streamKey); a holder asking for something else restarts the stream
|
||||
// rather than sharing it.
|
||||
private streamProvider: string | null = null;
|
||||
// The options of the running stream, for the description of a recording of it.
|
||||
private streamOptions: PushToTalkOptions | null = null;
|
||||
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
|
||||
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
|
||||
// controller runs in the decode-only "cloud" mode and every decoded PCM
|
||||
// frame is broadcast to these listeners. STT capture preempts it — a live
|
||||
@@ -150,11 +197,14 @@ export class FaceclawVoiceControlBridge {
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set: every source's chunks as they arrive, before any processing, written
|
||||
// by the controller. The file outlives a capture, so a restarted one keeps
|
||||
// adding to it, and it is created by the first audio, so a failed start
|
||||
// leaves no file.
|
||||
private recordingName = "";
|
||||
private recorder: WavRecorder | null = null;
|
||||
// The file is open in the controller.
|
||||
private recordingOpen = false;
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false;
|
||||
private readonly recordingListeners = new Set<() => void>();
|
||||
@@ -251,16 +301,55 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* End the continuous capture as an utterance: stop it and have the
|
||||
* transcript of its last audio come as a final. For a capture whose audio
|
||||
* ran out by itself (a replay).
|
||||
*/
|
||||
finishContinuousCapture(): void {
|
||||
this.releaseCapture("continuous", true);
|
||||
}
|
||||
|
||||
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
|
||||
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
|
||||
this.replayListeners.add(listener);
|
||||
return () => this.replayListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Continue the running replay from `positionMs` into its recording. */
|
||||
seekReplay(positionMs: number): void {
|
||||
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. */
|
||||
setReplayPaused(paused: boolean): void {
|
||||
this.controller?.setReplayPaused(paused);
|
||||
}
|
||||
|
||||
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
|
||||
isCaptureLive(): boolean {
|
||||
return this.micIsLive();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or "".
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return;
|
||||
this.recorder?.close();
|
||||
this.recorder = null;
|
||||
this.closeRecording();
|
||||
this.setRecording(name, false);
|
||||
}
|
||||
|
||||
private closeRecording(): void {
|
||||
if (!this.recordingOpen) return;
|
||||
this.recordingOpen = false;
|
||||
try {
|
||||
this.controller?.stopRecording();
|
||||
} catch (error) {
|
||||
console.warn(`Voice: closing the recording failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean {
|
||||
return this.recordingName !== "" && !this.recordingFailure;
|
||||
@@ -286,18 +375,48 @@ export class FaceclawVoiceControlBridge {
|
||||
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||
}
|
||||
|
||||
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||
private record(pcm: any): void {
|
||||
if (!this.recording) return;
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName);
|
||||
if (!this.recorder) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||
/**
|
||||
* Audio arrived for a continuous capture: open the recording asked for, if
|
||||
* it isn't open yet. The controller writes it from then on, as the audio
|
||||
* arrives on its own thread.
|
||||
*/
|
||||
private ensureRecording(): void {
|
||||
if (!this.recording || this.recordingOpen) return;
|
||||
const path = newRecordingPath(this.recordingName);
|
||||
let opened = false;
|
||||
try {
|
||||
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: could not create the recording: ${error}`);
|
||||
}
|
||||
this.recorder.append(pcm);
|
||||
if (!opened) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
this.recordingOpen = true;
|
||||
console.log(`Voice: recording audio to ${path}`);
|
||||
}
|
||||
|
||||
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
|
||||
private syncRecordingEnabled(): void {
|
||||
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
/** The running capture, as the JSON stored in a recording of it. */
|
||||
private describeCapture(): Record<string, unknown> {
|
||||
const options = this.streamOptions;
|
||||
return {
|
||||
recording: this.recordingName,
|
||||
startedAt: new Date().toISOString(),
|
||||
source: options?.usePhoneMic ? "phone" : "glasses",
|
||||
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
|
||||
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
|
||||
processing: {
|
||||
noiseSuppression: Boolean(options?.noiseSuppression),
|
||||
beamFilter: options?.beamFilter ?? null,
|
||||
},
|
||||
...options?.captureInfo,
|
||||
};
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
@@ -364,6 +483,8 @@ export class FaceclawVoiceControlBridge {
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
this.controller?.setMicArray(null);
|
||||
this.controller?.setReplay(null, 0);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
this.controller?.setNoiseSuppression(false);
|
||||
@@ -399,18 +520,33 @@ export class FaceclawVoiceControlBridge {
|
||||
const micLive = this.micIsLive();
|
||||
this.suspendedHolders.delete(holder);
|
||||
this.captureHolders.add(holder);
|
||||
if (micLive && this.streamProvider === transcriptionKey(options)) {
|
||||
if (micLive && this.streamProvider === streamKey(options)) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
this.syncRecordingEnabled();
|
||||
return;
|
||||
}
|
||||
// Any leftover capture is dead, or transcribes with another provider than
|
||||
// Any leftover capture is dead, or captures or transcribes otherwise than
|
||||
// this holder needs; drop it before starting the new one.
|
||||
if (this.started) this.teardownCapture();
|
||||
this.streamProvider = transcriptionKey(options);
|
||||
this.streamProvider = streamKey(options);
|
||||
this.streamOptions = options;
|
||||
this.ensureController();
|
||||
this.syncRecordingEnabled();
|
||||
// A recording under way carries on into this capture: note what changed.
|
||||
if (this.recordingOpen) {
|
||||
try {
|
||||
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: recording event failed: ${error}`);
|
||||
}
|
||||
}
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setMicArray(
|
||||
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
|
||||
);
|
||||
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
@@ -435,7 +571,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.speechPause.reset();
|
||||
// Replaces whatever the last capture left behind ("Listening...", an
|
||||
// error, the permission prompt) until the controller reports progress.
|
||||
this.setStatus("Starting microphone...");
|
||||
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
@@ -480,6 +616,7 @@ export class FaceclawVoiceControlBridge {
|
||||
if (commit) this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
this.syncRecordingEnabled();
|
||||
if (this.captureHolders.size > 0) {
|
||||
// Another holder still wants the mic; keep it running.
|
||||
return;
|
||||
@@ -504,6 +641,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.captureHolders.clear();
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
this.syncRecordingEnabled();
|
||||
this.teardownCapture();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
@@ -521,6 +659,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.suspendedHolders.add(holder);
|
||||
}
|
||||
this.captureHolders.clear();
|
||||
this.syncRecordingEnabled();
|
||||
this.suspendedRaw = this.rawActive;
|
||||
this.teardownCapture();
|
||||
if (this.suspendedHolders.size > 0) {
|
||||
@@ -578,6 +717,7 @@ export class FaceclawVoiceControlBridge {
|
||||
private teardownCapture(): void {
|
||||
this.rawActive = false;
|
||||
this.streamProvider = null;
|
||||
this.streamOptions = null;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
@@ -611,7 +751,7 @@ export class FaceclawVoiceControlBridge {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||
if (this.captureHolders.has("continuous")) this.ensureRecording();
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
@@ -642,6 +782,10 @@ export class FaceclawVoiceControlBridge {
|
||||
);
|
||||
}
|
||||
},
|
||||
onReplay: (kind: string, positionMs: number, durationMs: number) => {
|
||||
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
|
||||
for (const listener of Array.from(this.replayListeners)) listener(event);
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
@@ -678,6 +822,21 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture reads and transcribes with. A running capture is shared only
|
||||
* by holders asking for the same: the provider and language, the microphones
|
||||
* (the mixed stream, or the temples' own under a given configuration), or the
|
||||
* recording it replays.
|
||||
*/
|
||||
function streamKey(options: PushToTalkOptions): string {
|
||||
const source = options.replay
|
||||
? `replay:${options.replay.path}`
|
||||
: options.micArray && !options.usePhoneMic
|
||||
? `array:${Array.from(options.micArray.configure).join(",")}`
|
||||
: "mixed";
|
||||
return `${transcriptionKey(options)}|${source}`;
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
const javaFloats = Array.create("float", values.length);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
@@ -687,3 +846,136 @@ function toJavaFloats(values: Float32Array): any {
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
|
||||
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
|
||||
|
||||
/**
|
||||
* A recording played back as a capture of its own: transcribed by the same
|
||||
* pipeline as the microphone, with the same settings, on a bridge and native
|
||||
* controller separate from the microphone's. Nothing here touches the mic,
|
||||
* so voice input and a live Transcribe window carry on beside a replay.
|
||||
*/
|
||||
export class VoiceReplay {
|
||||
private readonly bridge = new FaceclawVoiceControlBridge();
|
||||
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
|
||||
private readonly offReplay: () => void;
|
||||
private position = 0;
|
||||
private duration = 0;
|
||||
private current: VoiceReplayState = "opening";
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* `options` gives the capture options (provider, keys, processing) each
|
||||
* time the replay starts, so a restart picks up changed settings.
|
||||
*/
|
||||
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
|
||||
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
|
||||
}
|
||||
|
||||
/** The replay's status, transcript and level events. */
|
||||
get events(): VoiceEventSource {
|
||||
return this.bridge;
|
||||
}
|
||||
|
||||
/** Milliseconds into the recording. */
|
||||
get positionMs(): number {
|
||||
return this.position;
|
||||
}
|
||||
|
||||
/** The recording's length in ms; 0 until it has been opened. */
|
||||
get durationMs(): number {
|
||||
return this.duration;
|
||||
}
|
||||
|
||||
get state(): VoiceReplayState {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/**
|
||||
* The position, duration or state changed. `jumpedToMs` is given when
|
||||
* playback moved there other than by playing on: a jump, or a start
|
||||
* part-way in. It comes after the transcript of what played before.
|
||||
*/
|
||||
onChange(listener: (jumpedToMs?: number) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Play from `positionMs` into the recording. */
|
||||
start(positionMs = 0): void {
|
||||
if (this.closed) return;
|
||||
this.position = Math.max(0, positionMs);
|
||||
this.set("opening");
|
||||
this.bridge.startContinuousCapture({
|
||||
...this.options(),
|
||||
communicator: null,
|
||||
usePhoneMic: false,
|
||||
micArray: null,
|
||||
replay: { path: this.path, startMs: this.position },
|
||||
});
|
||||
}
|
||||
|
||||
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
|
||||
jumpTo(positionMs: number): void {
|
||||
if (this.closed) return;
|
||||
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
|
||||
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
|
||||
this.bridge.seekReplay(target);
|
||||
this.position = target;
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.start(target);
|
||||
}
|
||||
|
||||
setPaused(paused: boolean): void {
|
||||
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
|
||||
this.bridge.setReplayPaused(paused);
|
||||
this.set(paused ? "paused" : "playing");
|
||||
}
|
||||
|
||||
/** Stop for good and release the capture. */
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.offReplay();
|
||||
this.listeners.clear();
|
||||
this.bridge.stop();
|
||||
}
|
||||
|
||||
private handle(event: VoiceReplayEvent): void {
|
||||
this.duration = event.durationMs;
|
||||
switch (event.kind) {
|
||||
case "start":
|
||||
this.position = event.positionMs;
|
||||
this.current = "playing";
|
||||
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
|
||||
return;
|
||||
case "seek":
|
||||
this.position = event.positionMs;
|
||||
this.notify(event.positionMs);
|
||||
return;
|
||||
case "end":
|
||||
this.position = event.durationMs;
|
||||
// Its last audio is still to be transcribed (a cloud provider's final).
|
||||
this.bridge.finishContinuousCapture();
|
||||
this.set("ended");
|
||||
return;
|
||||
case "error":
|
||||
this.set("failed");
|
||||
return;
|
||||
default:
|
||||
this.position = event.positionMs;
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private set(state: VoiceReplayState): void {
|
||||
this.current = state;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
private notify(jumpedToMs?: number): void {
|
||||
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
|
||||
export function wavHeader(dataBytes: number): Uint8Array {
|
||||
const header = new Uint8Array(44);
|
||||
const view = new DataView(header.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataBytes, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, 16000, true);
|
||||
view.setUint32(28, 32000, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, dataBytes, true);
|
||||
return header;
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
import { wavHeader } from './wav-header'
|
||||
|
||||
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
|
||||
const FLUSH_BYTES = 32000
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
|
||||
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private pending: Uint8Array[] = []
|
||||
private pendingBytes = 0
|
||||
private dataBytes = 0
|
||||
|
||||
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.wav`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
|
||||
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
|
||||
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
|
||||
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
|
||||
return new WavRecorder(handle, path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.handle || !pcm.length) return
|
||||
this.pending.push(pcm.slice())
|
||||
this.pendingBytes += pcm.length
|
||||
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.flush()
|
||||
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
|
||||
this.handle = null
|
||||
}
|
||||
|
||||
private flush(): void {
|
||||
const handle = this.handle
|
||||
if (!handle || !this.pendingBytes) return
|
||||
const chunk = new Uint8Array(this.pendingBytes)
|
||||
let offset = 0
|
||||
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
|
||||
this.pending = []
|
||||
this.pendingBytes = 0
|
||||
try {
|
||||
// NativeScript throws a failed call's NSError (storage full, file removed).
|
||||
handle.writeDataError(toData(chunk))
|
||||
handle.seekToOffsetError(0)
|
||||
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
|
||||
handle.seekToEndReturningOffsetError(null)
|
||||
this.dataBytes += chunk.length
|
||||
return
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
}
|
||||
try { handle.closeAndReturnError() } catch { /* ignore */ }
|
||||
this.handle = null
|
||||
}
|
||||
}
|
||||
|
||||
function toData(bytes: Uint8Array): NSData {
|
||||
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
declare const com: any;
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}.wav`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||
const path = String(file.getAbsolutePath());
|
||||
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||
} catch (error) {
|
||||
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||
append(pcm: any): void {
|
||||
this.recorder?.append(pcm);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.recorder?.finish();
|
||||
this.recorder = null;
|
||||
}
|
||||
}
|
||||
@@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
|
||||
storageKey: "developer.useMicControl",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
|
||||
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
|
||||
});
|
||||
|
||||
export const showBleBandwidthSetting = new ConfigSettingBoolean({
|
||||
@@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
|
||||
label: "Save Recording",
|
||||
storageKey: "voice.saveRecording",
|
||||
defaultValue: false,
|
||||
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
|
||||
description: global.isIOS
|
||||
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
|
||||
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
|
||||
+19
-1
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
|
||||
import { dimPlanes, type Plane } from "../graphics/plane";
|
||||
import { spanCurrent } from "../native/frame-timings";
|
||||
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
|
||||
import { type VoiceReplay } from "../native/voice-control";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
|
||||
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
|
||||
|
||||
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
|
||||
|
||||
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
||||
export type ContinuousCaptureOptions = {
|
||||
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
|
||||
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
|
||||
recordAs?: string;
|
||||
/**
|
||||
* How many of the glasses' microphones to capture: 1 is their own mixed
|
||||
* stream; 2 (one per temple) and 4 (both of each temple) stream the
|
||||
* microphones separately through the custom firmware, for recording and
|
||||
* array processing. While any client asks for more than one, every capture
|
||||
* uses the most asked for. Glasses without the microphone channel, and the
|
||||
* phone's microphone, give 1 whatever is asked.
|
||||
*/
|
||||
mics?: MicCount;
|
||||
/**
|
||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||
* or "" for the provider to detect it), instead of the Transcription
|
||||
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
|
||||
|
||||
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
|
||||
|
||||
export type MicCount = 1 | 2 | 4;
|
||||
|
||||
export type LayerActions = {
|
||||
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
|
||||
requestRender: () => void;
|
||||
@@ -55,6 +67,12 @@ export type LayerActions = {
|
||||
*/
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
/**
|
||||
* Open a recording (.mka) for playing back through transcription in place
|
||||
* of the microphone; the caller starts and closes it. Absent, or null,
|
||||
* where recordings can't be replayed.
|
||||
*/
|
||||
openVoiceReplay?: (path: string) => VoiceReplay | null;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
|
||||
import { truncateText } from "../graphics/textwrap";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
|
||||
import { lineStep } from "./metrics";
|
||||
import type { Edges } from "./viewport";
|
||||
|
||||
@@ -68,8 +68,9 @@ export class MicLevelMeter {
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
|
||||
/** `source` is the capture whose level to show: the microphone's unless given. */
|
||||
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
|
||||
this.unsubscribe = source.onMicLevel((level) => {
|
||||
this.peak = Math.max(this.peak, level);
|
||||
});
|
||||
this.timer = setInterval(() => {
|
||||
|
||||
@@ -48,6 +48,8 @@ export type InProcessWindowOptions = {
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** See ShellWindow.keepsScreenOn. */
|
||||
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||
/** See ShellWindow.restoresApp. */
|
||||
restoresApp?: boolean;
|
||||
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||
onForegroundChanged?: (foreground: boolean) => void;
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
@@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
keepsScreenOn: options.keepsScreenOn,
|
||||
restoresApp: options.restoresApp,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
|
||||
@@ -151,6 +151,13 @@ export type ShellWindow = {
|
||||
* draws this window's top bar.
|
||||
*/
|
||||
heightMode: WindowHeightMode;
|
||||
/**
|
||||
* False for a window that must not bring its app back when Faceclaw
|
||||
* restarts (the controller's open-apps restore relaunches apps, not
|
||||
* windows): one that shows something a plain launch of its app would not,
|
||||
* where the relaunch would do something else in its place.
|
||||
*/
|
||||
restoresApp?: boolean;
|
||||
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
|
||||
close?: () => void;
|
||||
drawIcon: ShellChromeWindow["drawIcon"];
|
||||
|
||||
Reference in new issue
Block a user