Files
faceclaw/app/native/cloud-stt-provider.ts
T
FilipAlexanderandClaude Opus 5.5 e2caf28392 Voice: self-hosted Whisper server provider; Transcribe on iOS
- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible
  Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests,
  with URL, model, language and key settings (Voice). Pause-ended spans are
  one request each, finalized in audio order, with provisional text while a
  span is open. See notes/whisper-server.md.
- Transcribe (and other continuous capture) works on iOS with a cloud or
  Whisper-server provider; push-to-talk shares a running capture.
- Transcribe saves each session's audio as transcribe-<time>.wav in Downloads
  (setting "Save Transcribe audio", on by default), written as it is captured
  so a crash leaves a playable file; a saved transcript uses the same stamp.
- Calculator stops continuous listening when its window closes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 16:43:18 +02:00

102 lines
3.9 KiB
TypeScript

import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control";
import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt";
/**
* The transcription-provider setting plus the keys the cloud providers need.
* Shared by the Android and iOS voice bridges.
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
type CloudProvider = {
name: string;
apiKey: (selection: VoiceProviderSelection) => string;
/** Whether the provider can run; defaults to having a key. */
configured?: (selection: VoiceProviderSelection) => boolean;
/** What to tell the user when it can't. */
missing?: string;
create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient;
finalTimeoutMs?: number;
};
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
elevenlabs: {
name: "ElevenLabs",
apiKey: (selection) => selection.elevenLabsApiKey,
create: (options) => new ElevenLabsSttClient(options),
},
soniox: {
name: "Soniox",
apiKey: (selection) => selection.sonioxApiKey,
create: (options) => new SonioxSttClient(options),
},
whisper: {
name: "OpenAI",
apiKey: (selection) => selection.openAiApiKey,
create: (options) => new OpenAiRealtimeSttClient(options),
},
"whisper-server": {
name: "Whisper server",
apiKey: (selection) => selection.whisperServerApiKey ?? "",
configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")),
missing: "No Whisper server URL set; using on-device voice.",
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
};
function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean {
return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim());
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
const provider = CLOUD_PROVIDERS[selection.provider];
return Boolean(provider && isConfigured(provider, selection));
}
/**
* The cloud provider for a capture, or null to transcribe on-device. A cloud
* provider whose API key is missing falls back to on-device (reported via
* onStatus) rather than failing the capture outright. `continuous` says
* whether the capture is open-ended (Transcribe), where pauses commit segments.
*/
export function createCloudSttClient(
selection: VoiceProviderSelection,
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): ReconnectingSttClient | null {
const provider = CLOUD_PROVIDERS[selection.provider];
if (!provider) return null;
if (!isConfigured(provider, selection)) {
options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`);
return null;
}
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
continuous,
provider.finalTimeoutMs,
);
}