forked from Blendman974/faceclaw
Translate app: sound level meter; translate unfinished segments
This commit is contained in:
1 parent
bf2addd287
commit
0b7d93a3f1
25 files changed
+679
-175
No files matched your search
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
};
|
||||
|
||||
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
create: (options, selection) => new WhisperServerSttClient(options, {
|
||||
baseUrl: selection.whisperServerUrl ?? "",
|
||||
model: selection.whisperServerModel ?? "",
|
||||
language: selection.whisperServerLanguage ?? "",
|
||||
language: selection.language ?? "",
|
||||
}),
|
||||
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
|
||||
},
|
||||
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture transcribes with: the provider, and for Whisper server the
|
||||
* language it is told to expect. A running capture is shared only by holders
|
||||
* asking for the same.
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
|
||||
return selection.provider === "whisper-server"
|
||||
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
|
||||
: selection.provider;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
@@ -105,7 +106,7 @@ export function createCloudSttClient(
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => provider.create(clientOptions, selection),
|
||||
{ ...options, apiKey },
|
||||
{ ...options, apiKey, language: (selection.language ?? "").trim() },
|
||||
continuous,
|
||||
provider.finalTimeoutMs,
|
||||
);
|
||||
|
||||
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
|
||||
|
||||
export type CloudSttOptions = {
|
||||
apiKey: string;
|
||||
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
|
||||
language?: string;
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
|
||||
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
// Without language_code the model detects the language, which can go
|
||||
// badly wrong on short or noisy segments (Japanese heard as Russian).
|
||||
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
function sessionConfig(language: string): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -209,7 +209,7 @@ function sessionConfig(): object {
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model: MODEL_ID },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
// A hint, not a restriction: other languages are still recognized.
|
||||
...(this.options.language ? { language_hints: [this.options.language] } : {}),
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
@@ -24,6 +25,7 @@ export class IosVoiceControlBridge {
|
||||
private readonly statuses = new Set<(state: VoiceControlState) => void>()
|
||||
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
|
||||
private readonly ends = new Set<() => void>()
|
||||
private readonly levels = new Set<(level: number) => void>()
|
||||
private status = 'Voice input ready.'
|
||||
private generation = 0
|
||||
private session: IosMicrophoneSession | null = null
|
||||
@@ -60,6 +62,8 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */
|
||||
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
@@ -69,6 +73,7 @@ export class IosVoiceControlBridge {
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
||||
this.cloud?.acceptPcm(bytes)
|
||||
if (this.continuous) this.record(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.native
|
||||
|
||||
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
|
||||
// stopped for its duration.
|
||||
private readonly rawPcmListeners = new Set<RawPcmListener>();
|
||||
private readonly frameMetaListeners = new Set<FrameMetaListener>();
|
||||
private readonly micLevelListeners = new Set<(level: number) => void>();
|
||||
private rawActive = false;
|
||||
// Who wants the raw tap. It runs while any owner does, so one owner letting
|
||||
// go (an EvenHub app backgrounding) doesn't silence another.
|
||||
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.rawPcmListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
|
||||
* each decoded audio chunk, about every 50 ms, from any capture: STT in
|
||||
* every mode, or the raw tap.
|
||||
*/
|
||||
onMicLevel(listener: (level: number) => void): () => void {
|
||||
this.micLevelListeners.add(listener);
|
||||
return () => this.micLevelListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
|
||||
onFrameMeta(listener: FrameMetaListener): () => void {
|
||||
this.frameMetaListeners.add(listener);
|
||||
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
if (this.micLevelListeners.size > 0) {
|
||||
const level = pcmLevel(bytes);
|
||||
this.micLevelListeners.forEach((listener) => listener(level));
|
||||
}
|
||||
},
|
||||
onFrameMeta: (angleDegrees: number, ssr: number) => {
|
||||
if (this.frameMetaListeners.size === 0) return;
|
||||
|
||||
Reference in new issue
Block a user