forked from Blendman974/faceclaw
Add OpenAI Whisper Realtime integration
This commit is contained in:
1 parent
161dbd104e
commit
008f2277ca
7 files changed
+320
-28
No files matched your search
@@ -63,6 +63,7 @@ import { APP_VIEWPORT } from "../ui/shell/geometry";
|
||||
import { type LayerActions } from "../ui/layers";
|
||||
import {
|
||||
elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting,
|
||||
nightscoutApiTokenSetting,
|
||||
firmwareDebugFlagsSetting,
|
||||
nightscoutSiteUrlSetting,
|
||||
@@ -676,6 +677,7 @@ class DashboardController {
|
||||
communicator: communicator.getNativeCommunicator(),
|
||||
provider: voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
endpointing,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper).
|
||||
* The voice bridge holds one of these while a cloud provider owns the
|
||||
* transcript; the Java controller then only decodes LC3 to PCM and hands it
|
||||
* over via acceptPcm.
|
||||
*/
|
||||
|
||||
export type CloudSttTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type CloudSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export interface CloudSttClient {
|
||||
/** Open whatever the provider needs before audio arrives. */
|
||||
start(): void;
|
||||
/** Feed PCM (16 kHz signed-16-bit LE). */
|
||||
acceptPcm(pcm: Uint8Array): void;
|
||||
/** End of utterance: produce a final transcript. */
|
||||
finish(): void;
|
||||
/** Abandon the session without finalizing. */
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
/** Sample rate of the PCM the G2 mic path produces, shared by all providers. */
|
||||
export const CLOUD_STT_SAMPLE_RATE = 16000;
|
||||
|
||||
declare const android: any;
|
||||
|
||||
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
|
||||
export function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
const javaBytes = Array.create("byte", bytes.length);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
javaBytes[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
declare const android: any;
|
||||
|
||||
/**
|
||||
* ElevenLabs realtime speech-to-text over WebSocket.
|
||||
@@ -165,13 +166,3 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
const javaBytes = Array.create("byte", bytes.length);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
javaBytes[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||
*
|
||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||
* turn detection is disabled and input_audio_buffer.commit is sent on button
|
||||
* release. Two impedance mismatches with our pipeline, both handled here:
|
||||
* the endpoint only accepts 24 kHz PCM (the G2 mic path is 16 kHz, so chunks
|
||||
* pass through a 2:3 linear-interpolation upsampler), and transcripts arrive
|
||||
* as DELTAS, which are accumulated into the full text our replace-semantics
|
||||
* listeners expect.
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
const MODEL_ID = "gpt-realtime-whisper";
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private readonly upsampler = new PcmUpsampler();
|
||||
private latestText = "";
|
||||
private committed = false;
|
||||
|
||||
constructor(private readonly options: OpenAiSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(
|
||||
WS_URL,
|
||||
this.listenerProxy,
|
||||
"Authorization",
|
||||
`Bearer ${this.options.apiKey}`,
|
||||
);
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Feed PCM (16 kHz signed-16-bit LE); upsampled to 24 kHz for the API. */
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.closed || pcm.length === 0) return;
|
||||
const resampled = this.upsampler.process(pcm);
|
||||
if (resampled.length === 0) return;
|
||||
const base64 = encodeBase64(resampled);
|
||||
if (this.open) {
|
||||
this.sendChunk(base64);
|
||||
} else {
|
||||
this.pendingChunks.push(base64);
|
||||
}
|
||||
}
|
||||
|
||||
/** End of utterance: commit the buffer for a final transcript. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
if (this.open) {
|
||||
this.sendChunk("__commit__");
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.committed = true;
|
||||
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
||||
return;
|
||||
}
|
||||
this.trySend(JSON.stringify({ type: "input_audio_buffer.append", audio: base64OrCommit }));
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
} catch (error) {
|
||||
console.warn("openai send failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(text: string): void {
|
||||
let message: any;
|
||||
try {
|
||||
message = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
switch (message?.type) {
|
||||
case "session.created":
|
||||
case "session.updated":
|
||||
this.options.onStatus("Listening (OpenAI)...");
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
this.latestText += String(message.delta ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
this.latestText = String(message.transcript ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
// The utterance is done; nothing further is coming after the commit.
|
||||
if (this.committed) this.stop();
|
||||
return;
|
||||
case "error":
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model: MODEL_ID },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming 16 kHz -> 24 kHz PCM16LE upsampler (linear interpolation, exact
|
||||
* 2:3 ratio). Keeps the last sample and fractional phase across chunks so
|
||||
* chunk boundaries don't glitch. Linear interpolation adds a little imaging
|
||||
* noise above 8 kHz, which is irrelevant to speech recognition.
|
||||
*/
|
||||
class PcmUpsampler {
|
||||
private prev = 0;
|
||||
private havePrev = false;
|
||||
// Output position between prev and the next input sample, in input-sample
|
||||
// units; advances by inputRate/outputRate per emitted sample.
|
||||
private phase = 0;
|
||||
|
||||
process(bytes: Uint8Array): Uint8Array {
|
||||
const sampleCount = bytes.length >> 1;
|
||||
const step = CLOUD_STT_SAMPLE_RATE / TARGET_SAMPLE_RATE;
|
||||
// Phase starts each input interval at < 1 and advances by 2/3, so the
|
||||
// inner loop emits at most 2 outputs per input sample.
|
||||
const out = new Uint8Array(sampleCount * 2 * 2);
|
||||
let outBytes = 0;
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
let sample = bytes[2 * i]! | (bytes[2 * i + 1]! << 8);
|
||||
if (sample >= 0x8000) sample -= 0x10000;
|
||||
if (!this.havePrev) {
|
||||
this.havePrev = true;
|
||||
this.prev = sample;
|
||||
continue;
|
||||
}
|
||||
while (this.phase < 1) {
|
||||
const value = Math.round(this.prev + this.phase * (sample - this.prev));
|
||||
out[outBytes++] = value & 0xff;
|
||||
out[outBytes++] = (value >> 8) & 0xff;
|
||||
this.phase += step;
|
||||
}
|
||||
this.phase -= 1;
|
||||
this.prev = sample;
|
||||
}
|
||||
return out.subarray(0, outBytes);
|
||||
}
|
||||
}
|
||||
+40
-14
@@ -1,6 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { CloudSttClient } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
@@ -9,7 +11,7 @@ export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs";
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
/**
|
||||
@@ -24,6 +26,7 @@ export type PushToTalkOptions = {
|
||||
communicator: any;
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
@@ -51,7 +54,7 @@ export class FaceclawVoiceControlBridge {
|
||||
// listener, so push-to-talk and the Transcribe window both receive text.
|
||||
private readonly captureHolders = new Set<CaptureHolder>();
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: ElevenLabsSttClient | null = null;
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -111,27 +114,50 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller?.setSaveRecordings(options.saveRecording);
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
|
||||
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
|
||||
this.cloudClient = new ElevenLabsSttClient({
|
||||
apiKey: options.elevenLabsApiKey.trim(),
|
||||
onTranscript: (event) => this.emitTranscript(event.text, event.isFinal),
|
||||
onStatus: (status) => this.setStatus(status),
|
||||
onError: (message) => this.setStatus(message),
|
||||
});
|
||||
this.cloudClient.start();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
if (cloudClient) {
|
||||
this.cloudClient = cloudClient;
|
||||
cloudClient.start();
|
||||
this.started = true;
|
||||
this.controller?.start("cloud");
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.provider === "elevenlabs") {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
}
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
this.controller?.start("onboard");
|
||||
}
|
||||
|
||||
/**
|
||||
* The cloud provider for this session, or null to transcribe on-device.
|
||||
* A cloud provider whose API key is missing falls back to on-device rather
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: { text: string; isFinal: boolean }) =>
|
||||
this.emitTranscript(event.text, event.isFinal),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
if (!this.captureHolders.delete(holder)) {
|
||||
// Never held; still finish a dangling cloud commit if one is pending.
|
||||
@@ -147,7 +173,7 @@ export class FaceclawVoiceControlBridge {
|
||||
return;
|
||||
}
|
||||
// Order matters for cloud: stopping the Java controller flushes any final
|
||||
// decode/PCM; then commit so ElevenLabs finalizes the transcript.
|
||||
// decode/PCM; then commit so the provider finalizes the transcript.
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
if (commit) {
|
||||
|
||||
@@ -233,15 +233,21 @@ export const firmwareDebugFlagsSetting = new ConfigSettingBoolean({
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export type VoiceProvider = "onboard" | "elevenlabs";
|
||||
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper";
|
||||
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "Whisper",
|
||||
};
|
||||
|
||||
export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
id: "voice-provider",
|
||||
label: "Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "elevenlabs"],
|
||||
formatValue: (value) => (value === "elevenlabs" ? "ElevenLabs" : "On-device"),
|
||||
values: ["onboard", "elevenlabs", "whisper"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
});
|
||||
|
||||
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
||||
@@ -261,6 +267,16 @@ export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
});
|
||||
|
||||
export const openAiApiKeySetting = new ConfigSettingString({
|
||||
id: "openai-api-key",
|
||||
label: "OpenAI key",
|
||||
storageKey: "voice.openAiApiKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "OpenAI API key",
|
||||
glassesEditTitle: "Edit OpenAI key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
});
|
||||
|
||||
export const anthropicApiKeySetting = new ConfigSettingString({
|
||||
id: "anthropic-api-key",
|
||||
label: "Anthropic key",
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
anthropicApiKeySetting,
|
||||
batteryDisplayModeSetting,
|
||||
elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting,
|
||||
enumSettingMenuItem,
|
||||
firmwareDebugFlagsSetting,
|
||||
rawScreenshotsEnabledSetting,
|
||||
@@ -54,6 +55,7 @@ function settingsSections(): SettingsSection[] {
|
||||
label: "API Keys",
|
||||
items: [
|
||||
textSettingMenuItem(elevenLabsApiKeySetting),
|
||||
textSettingMenuItem(openAiApiKeySetting),
|
||||
textSettingMenuItem(anthropicApiKeySetting),
|
||||
],
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user