diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 33a46f97..6e0786ee 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -63,6 +63,7 @@ import { APP_VIEWPORT } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; import { elevenLabsApiKeySetting, + openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, nightscoutSiteUrlSetting, @@ -676,6 +677,7 @@ class DashboardController { communicator: communicator.getNativeCommunicator(), provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), endpointing, }; diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts new file mode 100644 index 00000000..95c73a1d --- /dev/null +++ b/app/native/cloud-stt.ts @@ -0,0 +1,45 @@ +/** + * Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper). + * The voice bridge holds one of these while a cloud provider owns the + * transcript; the Java controller then only decodes LC3 to PCM and hands it + * over via acceptPcm. + */ + +export type CloudSttTranscriptEvent = { + text: string; + isFinal: boolean; +}; + +export type CloudSttOptions = { + apiKey: string; + onTranscript: (event: CloudSttTranscriptEvent) => void; + onStatus: (status: string) => void; + onError: (message: string) => void; +}; + +export interface CloudSttClient { + /** Open whatever the provider needs before audio arrives. */ + start(): void; + /** Feed PCM (16 kHz signed-16-bit LE). */ + acceptPcm(pcm: Uint8Array): void; + /** End of utterance: produce a final transcript. */ + finish(): void; + /** Abandon the session without finalizing. */ + stop(): void; +} + +/** Sample rate of the PCM the G2 mic path produces, shared by all providers. */ +export const CLOUD_STT_SAMPLE_RATE = 16000; + +declare const android: any; + +/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */ +export function encodeBase64(bytes: Uint8Array): string { + if (!global.isAndroid) return ""; + const javaBytes = Array.create("byte", bytes.length); + for (let i = 0; i < bytes.length; i++) { + const value = bytes[i]!; + javaBytes[i] = value > 127 ? value - 256 : value; + } + return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP)); +} diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index ab331083..8d602078 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -1,5 +1,6 @@ +import { encodeBase64 } from "./cloud-stt"; + declare const com: any; -declare const android: any; /** * ElevenLabs realtime speech-to-text over WebSocket. @@ -165,13 +166,3 @@ export class ElevenLabsSttClient { } } } - -function encodeBase64(bytes: Uint8Array): string { - if (!global.isAndroid) return ""; - const javaBytes = Array.create("byte", bytes.length); - for (let i = 0; i < bytes.length; i++) { - const value = bytes[i]!; - javaBytes[i] = value > 127 ? value - 256 : value; - } - return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP)); -} diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts new file mode 100644 index 00000000..e634f3cf --- /dev/null +++ b/app/native/openai-stt.ts @@ -0,0 +1,210 @@ +import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt"; + +declare const com: any; + +/** + * OpenAI realtime speech-to-text over WebSocket (a transcription-only session + * on the Realtime API, with the streaming gpt-realtime-whisper model). + * https://developers.openai.com/api/docs/guides/realtime-transcription + * + * Like the ElevenLabs client, push-to-talk defines the utterance boundary: + * turn detection is disabled and input_audio_buffer.commit is sent on button + * release. Two impedance mismatches with our pipeline, both handled here: + * the endpoint only accepts 24 kHz PCM (the G2 mic path is 16 kHz, so chunks + * pass through a 2:3 linear-interpolation upsampler), and transcripts arrive + * as DELTAS, which are accumulated into the full text our replace-semantics + * listeners expect. + */ + +const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription"; +const MODEL_ID = "gpt-realtime-whisper"; +const TARGET_SAMPLE_RATE = 24000; + +export type OpenAiSttOptions = CloudSttOptions; + +export class OpenAiRealtimeSttClient implements CloudSttClient { + private ws: any = null; + private listenerProxy: any = null; + private open = false; + private closed = false; + // Base64 audio (or the commit sentinel) queued until the socket opens. + private readonly pendingChunks: string[] = []; + private readonly upsampler = new PcmUpsampler(); + private latestText = ""; + private committed = false; + + constructor(private readonly options: OpenAiSttOptions) {} + + start(): void { + if (this.ws) return; + this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + onOpen: () => { + this.open = true; + this.trySend(JSON.stringify(sessionConfig())); + for (const chunk of this.pendingChunks.splice(0)) { + this.sendChunk(chunk); + } + }, + onTextMessage: (message: string) => this.handleMessage(String(message)), + onClosed: () => { + this.open = false; + }, + onFailure: (message: string) => { + if (this.closed) return; + this.options.onError(`OpenAI connection failed: ${String(message)}`); + }, + }); + try { + this.ws = new com.faceclaw.app.FaceclawWebSocket( + WS_URL, + this.listenerProxy, + "Authorization", + `Bearer ${this.options.apiKey}`, + ); + this.options.onStatus("Connecting to OpenAI..."); + } catch (error) { + this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`); + } + } + + /** Feed PCM (16 kHz signed-16-bit LE); upsampled to 24 kHz for the API. */ + acceptPcm(pcm: Uint8Array): void { + if (this.closed || pcm.length === 0) return; + const resampled = this.upsampler.process(pcm); + if (resampled.length === 0) return; + const base64 = encodeBase64(resampled); + if (this.open) { + this.sendChunk(base64); + } else { + this.pendingChunks.push(base64); + } + } + + /** End of utterance: commit the buffer for a final transcript. */ + finish(): void { + if (this.closed) return; + if (this.open) { + this.sendChunk("__commit__"); + } else { + this.pendingChunks.push("__commit__"); + } + } + + stop(): void { + this.closed = true; + if (this.ws) { + try { + this.ws.close(1000, "bye"); + } catch { + // ignore + } + this.ws = null; + } + this.listenerProxy = null; + } + + private sendChunk(base64OrCommit: string): void { + if (base64OrCommit === "__commit__") { + this.committed = true; + this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" })); + return; + } + this.trySend(JSON.stringify({ type: "input_audio_buffer.append", audio: base64OrCommit })); + } + + private trySend(message: string): void { + try { + this.ws?.sendText(message); + } catch (error) { + console.warn("openai send failed", error); + } + } + + private handleMessage(text: string): void { + let message: any; + try { + message = JSON.parse(text); + } catch { + return; + } + switch (message?.type) { + case "session.created": + case "session.updated": + this.options.onStatus("Listening (OpenAI)..."); + return; + case "conversation.item.input_audio_transcription.delta": + this.latestText += String(message.delta ?? ""); + this.options.onTranscript({ text: this.latestText, isFinal: false }); + return; + case "conversation.item.input_audio_transcription.completed": + this.latestText = String(message.transcript ?? this.latestText); + this.options.onTranscript({ text: this.latestText, isFinal: true }); + // The utterance is done; nothing further is coming after the commit. + if (this.committed) this.stop(); + return; + case "error": + this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`); + return; + default: + return; + } + } +} + +function sessionConfig(): object { + return { + type: "session.update", + session: { + type: "transcription", + audio: { + input: { + format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE }, + transcription: { model: MODEL_ID }, + // Push-to-talk owns the utterance boundary; no server VAD. + turn_detection: null, + }, + }, + }, + }; +} + +/** + * Streaming 16 kHz -> 24 kHz PCM16LE upsampler (linear interpolation, exact + * 2:3 ratio). Keeps the last sample and fractional phase across chunks so + * chunk boundaries don't glitch. Linear interpolation adds a little imaging + * noise above 8 kHz, which is irrelevant to speech recognition. + */ +class PcmUpsampler { + private prev = 0; + private havePrev = false; + // Output position between prev and the next input sample, in input-sample + // units; advances by inputRate/outputRate per emitted sample. + private phase = 0; + + process(bytes: Uint8Array): Uint8Array { + const sampleCount = bytes.length >> 1; + const step = CLOUD_STT_SAMPLE_RATE / TARGET_SAMPLE_RATE; + // Phase starts each input interval at < 1 and advances by 2/3, so the + // inner loop emits at most 2 outputs per input sample. + const out = new Uint8Array(sampleCount * 2 * 2); + let outBytes = 0; + for (let i = 0; i < sampleCount; i++) { + let sample = bytes[2 * i]! | (bytes[2 * i + 1]! << 8); + if (sample >= 0x8000) sample -= 0x10000; + if (!this.havePrev) { + this.havePrev = true; + this.prev = sample; + continue; + } + while (this.phase < 1) { + const value = Math.round(this.prev + this.phase * (sample - this.prev)); + out[outBytes++] = value & 0xff; + out[outBytes++] = (value >> 8) & 0xff; + this.phase += step; + } + this.phase -= 1; + this.prev = sample; + } + return out.subarray(0, outBytes); + } +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 9dc59009..21df7a1f 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,6 +1,8 @@ import { Utils } from "@nativescript/core"; +import { CloudSttClient } from "./cloud-stt"; import { ElevenLabsSttClient } from "./elevenlabs-stt"; +import { OpenAiRealtimeSttClient } from "./openai-stt"; import { toUint8Array } from "../util/array-util"; declare const com: any; @@ -9,7 +11,7 @@ export type VoiceControlState = { status: string; }; -export type VoiceProviderKind = "onboard" | "elevenlabs"; +export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper"; export type VoiceTranscriptEvent = { /** @@ -24,6 +26,7 @@ export type PushToTalkOptions = { communicator: any; provider: VoiceProviderKind; elevenLabsApiKey: string; + openAiApiKey: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free @@ -51,7 +54,7 @@ export class FaceclawVoiceControlBridge { // listener, so push-to-talk and the Transcribe window both receive text. private readonly captureHolders = new Set(); // Non-null while a cloud provider owns the transcript; Java only decodes PCM. - private cloudClient: ElevenLabsSttClient | null = null; + private cloudClient: CloudSttClient | null = null; onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); @@ -111,27 +114,50 @@ export class FaceclawVoiceControlBridge { this.controller?.setSaveRecordings(options.saveRecording); this.controller?.setEndpointing(Boolean(options.endpointing)); - if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) { - this.cloudClient = new ElevenLabsSttClient({ - apiKey: options.elevenLabsApiKey.trim(), - onTranscript: (event) => this.emitTranscript(event.text, event.isFinal), - onStatus: (status) => this.setStatus(status), - onError: (message) => this.setStatus(message), - }); - this.cloudClient.start(); + const cloudClient = this.createCloudClient(options); + if (cloudClient) { + this.cloudClient = cloudClient; + cloudClient.start(); this.started = true; this.controller?.start("cloud"); return; } - if (options.provider === "elevenlabs") { - this.setStatus("No ElevenLabs key set; using on-device voice."); - } this.cloudClient = null; this.started = true; this.controller?.start("onboard"); } + /** + * The cloud provider for this session, or null to transcribe on-device. + * A cloud provider whose API key is missing falls back to on-device rather + * than failing the capture outright. + */ + private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { + if (options.provider === "onboard") return null; + const sttOptions = { + apiKey: "", + onTranscript: (event: { text: string; isFinal: boolean }) => + this.emitTranscript(event.text, event.isFinal), + onStatus: (status: string) => this.setStatus(status), + onError: (message: string) => this.setStatus(message), + }; + if (options.provider === "elevenlabs") { + const apiKey = options.elevenLabsApiKey.trim(); + if (!apiKey) { + this.setStatus("No ElevenLabs key set; using on-device voice."); + return null; + } + return new ElevenLabsSttClient({ ...sttOptions, apiKey }); + } + const apiKey = options.openAiApiKey.trim(); + if (!apiKey) { + this.setStatus("No OpenAI key set; using on-device voice."); + return null; + } + return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey }); + } + private releaseCapture(holder: CaptureHolder, commit: boolean): void { if (!this.captureHolders.delete(holder)) { // Never held; still finish a dangling cloud commit if one is pending. @@ -147,7 +173,7 @@ export class FaceclawVoiceControlBridge { return; } // Order matters for cloud: stopping the Java controller flushes any final - // decode/PCM; then commit so ElevenLabs finalizes the transcript. + // decode/PCM; then commit so the provider finalizes the transcript. this.controller?.stop(); this.started = false; if (commit) { diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 01378b94..04ffcfdc 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -233,15 +233,21 @@ export const firmwareDebugFlagsSetting = new ConfigSettingBoolean({ defaultValue: false, }); -export type VoiceProvider = "onboard" | "elevenlabs"; +export type VoiceProvider = "onboard" | "elevenlabs" | "whisper"; + +const voiceProviderLabels: Record = { + onboard: "On-device", + elevenlabs: "ElevenLabs", + whisper: "Whisper", +}; export const voiceProviderSetting = new ConfigSettingEnum({ id: "voice-provider", label: "Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: ["onboard", "elevenlabs"], - formatValue: (value) => (value === "elevenlabs" ? "ElevenLabs" : "On-device"), + values: ["onboard", "elevenlabs", "whisper"], + formatValue: (value) => voiceProviderLabels[value] ?? value, }); export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({ @@ -261,6 +267,16 @@ export const elevenLabsApiKeySetting = new ConfigSettingString({ formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), }); +export const openAiApiKeySetting = new ConfigSettingString({ + id: "openai-api-key", + label: "OpenAI key", + storageKey: "voice.openAiApiKey", + defaultValue: "", + editorTitle: "OpenAI API key", + glassesEditTitle: "Edit OpenAI key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), +}); + export const anthropicApiKeySetting = new ConfigSettingString({ id: "anthropic-api-key", label: "Anthropic key", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 3a87d19f..2f07af34 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -5,6 +5,7 @@ import { anthropicApiKeySetting, batteryDisplayModeSetting, elevenLabsApiKeySetting, + openAiApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, @@ -54,6 +55,7 @@ function settingsSections(): SettingsSection[] { label: "API Keys", items: [ textSettingMenuItem(elevenLabsApiKeySetting), + textSettingMenuItem(openAiApiKeySetting), textSettingMenuItem(anthropicApiKeySetting), ], },