forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw
This commit is contained in:
commit
c162c8e633
24 files changed
+486
-154
No files matched your search
@@ -62,6 +62,9 @@ export class IosVoiceControlBridge {
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false
|
||||
private readonly recordingListeners = new Set<() => void>()
|
||||
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||
// rawListeners while any owner wants it, from the session setRawSession
|
||||
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||
@@ -212,16 +215,28 @@ export class IosVoiceControlBridge {
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return
|
||||
this.recorder?.close(); this.recorder = null
|
||||
this.recordingName = name
|
||||
this.setRecording(name, false)
|
||||
}
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean { return this.recordingName !== '' && !this.recordingFailure }
|
||||
/** Whether the recording asked for could not be saved (its file could not be created). */
|
||||
get recordingFailed(): boolean { return this.recordingFailure }
|
||||
/** `recording` or `recordingFailed` changed. */
|
||||
onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => { this.recordingListeners.delete(listener) } }
|
||||
private setRecording(name: string, failed: boolean): void {
|
||||
const wasRecording = this.recording, hadFailed = this.recordingFailure
|
||||
this.recordingName = name; this.recordingFailure = failed
|
||||
if (wasRecording === this.recording && hadFailed === failed) return
|
||||
for (const fn of [...this.recordingListeners]) fn()
|
||||
}
|
||||
private record(pcm: Uint8Array): void {
|
||||
if (!this.recorder && this.recordingName) {
|
||||
if (!this.recording) return
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName)
|
||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
// One attempt per recording, not one per packet.
|
||||
if (!this.recorder) this.recordingName = ''
|
||||
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||
}
|
||||
this.recorder?.append(pcm)
|
||||
this.recorder.append(pcm)
|
||||
}
|
||||
/** End continuous capture, transcribing the audio since the last pause. */
|
||||
stopContinuousCapture(): Promise<void> {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { WavRecorder } from "./wav-recorder";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
@@ -60,7 +61,6 @@ export type PushToTalkOptions = {
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
* ("Hey Even") capture, which has no button release to end the utterance.
|
||||
@@ -150,6 +150,14 @@ export class FaceclawVoiceControlBridge {
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = "";
|
||||
private recorder: WavRecorder | null = null;
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false;
|
||||
private readonly recordingListeners = new Set<() => void>();
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -242,6 +250,56 @@ export class FaceclawVoiceControlBridge {
|
||||
this.acquireCapture("continuous", options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* until called with another name or "".
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return;
|
||||
this.recorder?.close();
|
||||
this.recorder = null;
|
||||
this.setRecording(name, false);
|
||||
}
|
||||
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean {
|
||||
return this.recordingName !== "" && !this.recordingFailure;
|
||||
}
|
||||
|
||||
/** Whether the recording asked for could not be saved (its file could not be created). */
|
||||
get recordingFailed(): boolean {
|
||||
return this.recordingFailure;
|
||||
}
|
||||
|
||||
/** `recording` or `recordingFailed` changed. */
|
||||
onRecordingChange(listener: () => void): () => void {
|
||||
this.recordingListeners.add(listener);
|
||||
return () => this.recordingListeners.delete(listener);
|
||||
}
|
||||
|
||||
private setRecording(name: string, failed: boolean): void {
|
||||
const wasRecording = this.recording;
|
||||
const hadFailed = this.recordingFailure;
|
||||
this.recordingName = name;
|
||||
this.recordingFailure = failed;
|
||||
if (wasRecording === this.recording && hadFailed === failed) return;
|
||||
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||
}
|
||||
|
||||
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||
private record(pcm: any): void {
|
||||
if (!this.recording) return;
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName);
|
||||
if (!this.recorder) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||
}
|
||||
this.recorder.append(pcm);
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
onRawPcm(listener: RawPcmListener): () => void {
|
||||
this.rawPcmListeners.add(listener);
|
||||
@@ -305,7 +363,6 @@ export class FaceclawVoiceControlBridge {
|
||||
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setSaveRecordings(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
@@ -354,7 +411,6 @@ export class FaceclawVoiceControlBridge {
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setSaveRecordings(options.saveRecording);
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
@@ -554,6 +610,8 @@ export class FaceclawVoiceControlBridge {
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
declare const com: any;
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}.wav`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||
const path = String(file.getAbsolutePath());
|
||||
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||
} catch (error) {
|
||||
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||
append(pcm: any): void {
|
||||
this.recorder?.append(pcm);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.recorder?.finish();
|
||||
this.recorder = null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user