forked from Blendman974/faceclaw
Multi-mic support and replay mode for Transcribe app
This commit is contained in:
1 parent
c162c8e633
commit
9372b2742c
45 files changed
+3926
-288
No files matched your search
@@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
|
||||
storageKey: "developer.useMicControl",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
|
||||
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
|
||||
});
|
||||
|
||||
export const showBleBandwidthSetting = new ConfigSettingBoolean({
|
||||
@@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
|
||||
label: "Save Recording",
|
||||
storageKey: "voice.saveRecording",
|
||||
defaultValue: false,
|
||||
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
|
||||
description: global.isIOS
|
||||
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
|
||||
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
|
||||
+19
-1
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
|
||||
import { dimPlanes, type Plane } from "../graphics/plane";
|
||||
import { spanCurrent } from "../native/frame-timings";
|
||||
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
|
||||
import { type VoiceReplay } from "../native/voice-control";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
|
||||
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
|
||||
|
||||
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
|
||||
|
||||
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
||||
export type ContinuousCaptureOptions = {
|
||||
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
|
||||
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
|
||||
recordAs?: string;
|
||||
/**
|
||||
* How many of the glasses' microphones to capture: 1 is their own mixed
|
||||
* stream; 2 (one per temple) and 4 (both of each temple) stream the
|
||||
* microphones separately through the custom firmware, for recording and
|
||||
* array processing. While any client asks for more than one, every capture
|
||||
* uses the most asked for. Glasses without the microphone channel, and the
|
||||
* phone's microphone, give 1 whatever is asked.
|
||||
*/
|
||||
mics?: MicCount;
|
||||
/**
|
||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||
* or "" for the provider to detect it), instead of the Transcription
|
||||
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
|
||||
|
||||
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
|
||||
|
||||
export type MicCount = 1 | 2 | 4;
|
||||
|
||||
export type LayerActions = {
|
||||
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
|
||||
requestRender: () => void;
|
||||
@@ -55,6 +67,12 @@ export type LayerActions = {
|
||||
*/
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
/**
|
||||
* Open a recording (.mka) for playing back through transcription in place
|
||||
* of the microphone; the caller starts and closes it. Absent, or null,
|
||||
* where recordings can't be replayed.
|
||||
*/
|
||||
openVoiceReplay?: (path: string) => VoiceReplay | null;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
|
||||
import { truncateText } from "../graphics/textwrap";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
|
||||
import { lineStep } from "./metrics";
|
||||
import type { Edges } from "./viewport";
|
||||
|
||||
@@ -68,8 +68,9 @@ export class MicLevelMeter {
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
|
||||
/** `source` is the capture whose level to show: the microphone's unless given. */
|
||||
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
|
||||
this.unsubscribe = source.onMicLevel((level) => {
|
||||
this.peak = Math.max(this.peak, level);
|
||||
});
|
||||
this.timer = setInterval(() => {
|
||||
|
||||
@@ -48,6 +48,8 @@ export type InProcessWindowOptions = {
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** See ShellWindow.keepsScreenOn. */
|
||||
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||
/** See ShellWindow.restoresApp. */
|
||||
restoresApp?: boolean;
|
||||
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||
onForegroundChanged?: (foreground: boolean) => void;
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
@@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
keepsScreenOn: options.keepsScreenOn,
|
||||
restoresApp: options.restoresApp,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
|
||||
@@ -151,6 +151,13 @@ export type ShellWindow = {
|
||||
* draws this window's top bar.
|
||||
*/
|
||||
heightMode: WindowHeightMode;
|
||||
/**
|
||||
* False for a window that must not bring its app back when Faceclaw
|
||||
* restarts (the controller's open-apps restore relaunches apps, not
|
||||
* windows): one that shows something a plain launch of its app would not,
|
||||
* where the relaunch would do something else in its place.
|
||||
*/
|
||||
restoresApp?: boolean;
|
||||
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
|
||||
close?: () => void;
|
||||
drawIcon: ShellChromeWindow["drawIcon"];
|
||||
|
||||
Reference in new issue
Block a user