Translate app: sound level meter; translate unfinished segments

This commit is contained in:
James Babcock committed 2026-10-25 13:12:25 -07:00
1 parent bf2addd287
commit 0b7d93a3f1
25 files changed
+679 -175

No files matched your search

+11 -10
View File
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
/**
* Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Cloud providers only.
*/
language?: string;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
language: selection.language ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
}
/**
* What a capture transcribes with: the provider, and for Whisper server the
* language it is told to expect. A running capture is shared only by holders
* asking for the same.
* What a capture transcribes with: the provider and the language it is told
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
return selection.provider === "whisper-server"
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
: selection.provider;
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
@@ -105,7 +106,7 @@ export function createCloudSttClient(
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
{ ...options, apiKey, language: (selection.language ?? "").trim() },
continuous,
provider.finalTimeoutMs,
);
+2
View File
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
export type CloudSttOptions = {
apiKey: string;
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
language?: string;
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
+4 -1
View File
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
start(): void {
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
// Without language_code the model detects the language, which can go
// badly wrong on short or noisy segments (Japanese heard as Russian).
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
const listener = {
onOpen: () => {
if (this.closed) return;
+3 -3
View File
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
function sessionConfig(): object {
function sessionConfig(language: string): object {
return {
type: "session.update",
session: {
@@ -209,7 +209,7 @@ function sessionConfig(): object {
audio: {
input: {
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
transcription: { model: MODEL_ID },
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
// Push-to-talk owns the utterance boundary; no server VAD.
turn_detection: null,
},
+2
View File
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
// A hint, not a restriction: other languages are still recognized.
...(this.options.language ? { language_hints: [this.options.language] } : {}),
}),
);
if (this.closed) return;
+5
View File
@@ -3,6 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { WavRecorder } from './wav-recorder.ios'
import { pcmLevel } from '../util/audio-level'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
@@ -24,6 +25,7 @@ export class IosVoiceControlBridge {
private readonly statuses = new Set<(state: VoiceControlState) => void>()
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
private readonly ends = new Set<() => void>()
private readonly levels = new Set<(level: number) => void>()
private status = 'Voice input ready.'
private generation = 0
private session: IosMicrophoneSession | null = null
@@ -60,6 +62,8 @@ export class IosVoiceControlBridge {
}
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} }
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
private ensureNative(): any {
if (!this.native) {
@@ -69,6 +73,7 @@ export class IosVoiceControlBridge {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
this.cloud?.acceptPcm(bytes)
if (this.continuous) this.record(bytes)
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
}
}
return this.native
+18 -1
View File
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
declare const com: any;
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
// stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private readonly micLevelListeners = new Set<(level: number) => void>();
private rawActive = false;
// Who wants the raw tap. It runs while any owner does, so one owner letting
// go (an EvenHub app backgrounding) doesn't silence another.
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
return () => this.rawPcmListeners.delete(listener);
}
/**
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
* each decoded audio chunk, about every 50 ms, from any capture: STT in
* every mode, or the raw tap.
*/
onMicLevel(listener: (level: number) => void): () => void {
this.micLevelListeners.add(listener);
return () => this.micLevelListeners.delete(listener);
}
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
onFrameMeta(listener: FrameMetaListener): () => void {
this.frameMetaListeners.add(listener);
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
if (this.micLevelListeners.size > 0) {
const level = pcmLevel(bytes);
this.micLevelListeners.forEach((listener) => listener(level));
}
},
onFrameMeta: (angleDegrees: number, ssr: number) => {
if (this.frameMetaListeners.size === 0) return;