Multi-mic support and replay mode for Transcribe app

This commit is contained in:
James Babcock committed 2026-11-12 21:54:57 -08:00
1 parent c162c8e633
commit 9372b2742c
45 files changed
+3926 -288

No files matched your search

+4 -2
View File
@@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
storageKey: "developer.useMicControl",
defaultValue: true,
description:
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
});
export const showBleBandwidthSetting = new ConfigSettingBoolean({
@@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
label: "Save Recording",
storageKey: "voice.saveRecording",
defaultValue: false,
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
description: global.isIOS
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
});
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
+19 -1
View File
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
import { dimPlanes, type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
import { type VoiceReplay } from "../native/voice-control";
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
recordAs?: string;
/**
* How many of the glasses' microphones to capture: 1 is their own mixed
* stream; 2 (one per temple) and 4 (both of each temple) stream the
* microphones separately through the custom firmware, for recording and
* array processing. While any client asks for more than one, every capture
* uses the most asked for. Glasses without the microphone channel, and the
* phone's microphone, give 1 whatever is asked.
*/
mics?: MicCount;
/**
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
* or "" for the provider to detect it), instead of the Transcription
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
export type MicCount = 1 | 2 | 4;
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
@@ -55,6 +67,12 @@ export type LayerActions = {
*/
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
/**
* Open a recording (.mka) for playing back through transcription in place
* of the microphone; the caller starts and closes it. Absent, or null,
* where recordings can't be replayed.
*/
openVoiceReplay?: (path: string) => VoiceReplay | null;
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
};
+4 -3
View File
@@ -1,6 +1,6 @@
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control";
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
import { lineStep } from "./metrics";
import type { Edges } from "./viewport";
@@ -68,8 +68,9 @@ export class MicLevelMeter {
private timer: ReturnType<typeof setInterval> | null = null;
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void, isVisible: () => boolean): void {
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
/** `source` is the capture whose level to show: the microphone's unless given. */
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
this.unsubscribe = source.onMicLevel((level) => {
this.peak = Math.max(this.peak, level);
});
this.timer = setInterval(() => {
+3
View File
@@ -48,6 +48,8 @@ export type InProcessWindowOptions = {
setScreenOn?: ShellWindow["setScreenOn"];
/** See ShellWindow.keepsScreenOn. */
keepsScreenOn?: ShellWindow["keepsScreenOn"];
/** See ShellWindow.restoresApp. */
restoresApp?: boolean;
/** The window came to or left the foreground (after its surface visibility follows). */
onForegroundChanged?: (foreground: boolean) => void;
/** Shared actions; requestRender is rebound to this window's render. */
@@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
keepsScreenOn: options.keepsScreenOn,
restoresApp: options.restoresApp,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
+7
View File
@@ -151,6 +151,13 @@ export type ShellWindow = {
* draws this window's top bar.
*/
heightMode: WindowHeightMode;
/**
* False for a window that must not bring its app back when Faceclaw
* restarts (the controller's open-apps restore relaunches apps, not
* windows): one that shows something a plain launch of its app would not,
* where the relaunch would do something else in its place.
*/
restoresApp?: boolean;
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
close?: () => void;
drawIcon: ShellChromeWindow["drawIcon"];