forked from Blendman974/faceclaw
Merge configurable OpenAI transcription model
This commit is contained in:
11 files changed
+113
-17
No files matched your search
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
|||||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||||
import {
|
import {
|
||||||
getInstalledEvenHubAppById,
|
getInstalledEvenHubAppById,
|
||||||
@@ -2266,7 +2266,7 @@ class DashboardController {
|
|||||||
provider,
|
provider,
|
||||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||||
openAiApiKey: openAiApiKeySetting.get(),
|
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
||||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||||
whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerModel: whisperServerModelSetting.get(),
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
|||||||
import { onAnySettingChanged,
|
import { onAnySettingChanged,
|
||||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
||||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||||
@@ -1211,7 +1211,7 @@ export class IosPreviewController {
|
|||||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||||
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
||||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||||
import { SonioxSttClient } from "./soniox-stt";
|
import { SonioxSttClient } from "./soniox-stt";
|
||||||
import type { VoiceProviderKind } from "./voice-control";
|
import type { VoiceProviderKind } from "./voice-control";
|
||||||
@@ -19,6 +19,7 @@ export type VoiceProviderSelection = {
|
|||||||
language?: string;
|
language?: string;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
|
openAiTranscriptionModel?: OpenAiSttModel;
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
@@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
|||||||
whisper: {
|
whisper: {
|
||||||
name: "OpenAI",
|
name: "OpenAI",
|
||||||
apiKey: (selection) => selection.openAiApiKey,
|
apiKey: (selection) => selection.openAiApiKey,
|
||||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
||||||
},
|
},
|
||||||
"whisper-server": {
|
"whisper-server": {
|
||||||
name: "Whisper server",
|
name: "Whisper server",
|
||||||
@@ -76,9 +77,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
|||||||
* What a capture transcribes with: the provider and the language it is told
|
* What a capture transcribes with: the provider and the language it is told
|
||||||
* to expect. A running capture is shared only by holders asking for the same.
|
* to expect. A running capture is shared only by holders asking for the same.
|
||||||
*/
|
*/
|
||||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||||
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
|
||||||
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`;
|
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||||
|
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
* on the Realtime API, with a configurable transcription model).
|
||||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||||
*
|
*
|
||||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||||
@@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket";
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||||
const MODEL_ID = "gpt-realtime-whisper";
|
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
||||||
|
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
||||||
const TARGET_SAMPLE_RATE = 24000;
|
const TARGET_SAMPLE_RATE = 24000;
|
||||||
|
|
||||||
export type OpenAiSttOptions = CloudSttOptions;
|
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
||||||
|
|
||||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||||
private ws: SocketConnection | null = null;
|
private ws: SocketConnection | null = null;
|
||||||
@@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
onOpen: () => {
|
onOpen: () => {
|
||||||
if (this.closed) return;
|
if (this.closed) return;
|
||||||
this.open = true;
|
this.open = true;
|
||||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
||||||
for (const chunk of this.pendingChunks.splice(0)) {
|
for (const chunk of this.pendingChunks.splice(0)) {
|
||||||
this.sendChunk(chunk);
|
this.sendChunk(chunk);
|
||||||
}
|
}
|
||||||
@@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
this.options.onError(message.error?.code === "model_not_found"
|
||||||
|
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
||||||
|
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||||
return;
|
return;
|
||||||
default:
|
default:
|
||||||
return;
|
return;
|
||||||
@@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function sessionConfig(language: string): object {
|
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
||||||
return {
|
return {
|
||||||
type: "session.update",
|
type: "session.update",
|
||||||
session: {
|
session: {
|
||||||
@@ -209,7 +212,7 @@ function sessionConfig(language: string): object {
|
|||||||
audio: {
|
audio: {
|
||||||
input: {
|
input: {
|
||||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
transcription: { model, ...(language ? { language } : {}) },
|
||||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||||
turn_detection: null,
|
turn_detection: null,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { OpenAiSttModel } from "./openai-stt";
|
||||||
import { Utils } from "@nativescript/core";
|
import { Utils } from "@nativescript/core";
|
||||||
|
|
||||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||||
@@ -24,7 +25,7 @@ export type VoiceControlState = {
|
|||||||
detail: string;
|
detail: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
||||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||||
@@ -54,6 +55,7 @@ export type PushToTalkOptions = {
|
|||||||
provider: VoiceProviderKind;
|
provider: VoiceProviderKind;
|
||||||
elevenLabsApiKey: string;
|
elevenLabsApiKey: string;
|
||||||
openAiApiKey: string;
|
openAiApiKey: string;
|
||||||
|
openAiTranscriptionModel?: OpenAiSttModel;
|
||||||
sonioxApiKey: string;
|
sonioxApiKey: string;
|
||||||
whisperServerUrl?: string;
|
whisperServerUrl?: string;
|
||||||
whisperServerModel?: string;
|
whisperServerModel?: string;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { readHealthExportSecret, writeHealthExportSecret } from "../native/health-export-secret";
|
import { readHealthExportSecret, writeHealthExportSecret } from "../native/health-export-secret";
|
||||||
|
import type { OpenAiSttModel } from "../native/openai-stt";
|
||||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||||
@@ -1251,6 +1252,15 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
|||||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
||||||
|
id: "openai-transcription-model",
|
||||||
|
label: "OpenAI transcription model",
|
||||||
|
storageKey: "voice.openAiTranscriptionModel",
|
||||||
|
defaultValue: "gpt-realtime-whisper",
|
||||||
|
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
||||||
|
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
||||||
|
});
|
||||||
|
|
||||||
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
||||||
id: "voice-language",
|
id: "voice-language",
|
||||||
label: "Transcription language",
|
label: "Transcription language",
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ import {
|
|||||||
mapboxApiKeySetting,
|
mapboxApiKeySetting,
|
||||||
mirrorTouchSetting,
|
mirrorTouchSetting,
|
||||||
openAiApiKeySetting,
|
openAiApiKeySetting,
|
||||||
|
openAiTranscriptionModelSetting,
|
||||||
previewColorSetting,
|
previewColorSetting,
|
||||||
previewScalingSetting,
|
previewScalingSetting,
|
||||||
phoneRotationSetting,
|
phoneRotationSetting,
|
||||||
@@ -549,6 +550,7 @@ function asrModelStatusText(id: AsrModelId): string {
|
|||||||
*/
|
*/
|
||||||
function whisperServerMenuItems(): MenuItem[] {
|
function whisperServerMenuItems(): MenuItem[] {
|
||||||
return [
|
return [
|
||||||
|
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
||||||
textSettingMenuItem(whisperServerUrlSetting),
|
textSettingMenuItem(whisperServerUrlSetting),
|
||||||
textSettingMenuItem(whisperServerModelSetting),
|
textSettingMenuItem(whisperServerModelSetting),
|
||||||
textSettingMenuItem(whisperServerLanguageSetting),
|
textSettingMenuItem(whisperServerLanguageSetting),
|
||||||
|
|||||||
@@ -44,3 +44,25 @@ software checks above; recognition accuracy and latency require device testing.
|
|||||||
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||||
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
||||||
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
||||||
|
|
||||||
|
## OpenAI transcription model selection
|
||||||
|
|
||||||
|
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
||||||
|
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
||||||
|
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
||||||
|
The unset default retains gpt-realtime-whisper for existing installations; no
|
||||||
|
provider, language or Assistant model preference is overwritten. This setting
|
||||||
|
applies to Translate's speech recognition as well as Transcribe and dictation.
|
||||||
|
The two transcribe alternatives produce transcript deltas after a committed
|
||||||
|
audio segment; existing pause segmentation and push-to-talk release supply the
|
||||||
|
commits. Choosing a model retargets active continuous capture.
|
||||||
|
|
||||||
|
Model access errors name the requested model before the long project error and
|
||||||
|
point to the setting. Factory/session-wire tests cover model, language, manual
|
||||||
|
commit and final transcript behavior; settings tests cover persistence and the
|
||||||
|
Android/iOS option plumbing. No real API inference was performed; actual project
|
||||||
|
access, recognition and latency remain to be checked on a device. The model list
|
||||||
|
provided by the operator establishes account visibility, not a successful audio
|
||||||
|
request.
|
||||||
|
|
||||||
|
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
||||||
@@ -242,3 +242,43 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
|||||||
assert.equal(transcription.language, language || undefined);
|
assert.equal(transcription.language, language || undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
||||||
|
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
||||||
|
const h = harness('openai', true, { model, language: 'fr' });
|
||||||
|
h.open();
|
||||||
|
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
||||||
|
assert.equal(input.transcription.model, model);
|
||||||
|
assert.equal(input.transcription.language, 'fr');
|
||||||
|
assert.equal(input.turn_detection, null);
|
||||||
|
h.client.acceptPcm(pcm(1200, 150));
|
||||||
|
h.client.acceptPcm(pcm(0, 1500));
|
||||||
|
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
||||||
|
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
||||||
|
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
||||||
|
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
||||||
|
assert.equal(h.events.at(-1).isFinal, true);
|
||||||
|
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
||||||
|
assert.ok(h.errors[0].includes(model));
|
||||||
|
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
||||||
|
const created = [];
|
||||||
|
const load = loader({ setTimeout, clearTimeout }, {
|
||||||
|
'./openai-stt': {
|
||||||
|
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
||||||
|
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
||||||
|
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
||||||
|
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
||||||
|
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
||||||
|
assert.equal(created[0].language, 'fr');
|
||||||
|
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||||
|
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||||
|
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
||||||
|
});
|
||||||
@@ -7,7 +7,7 @@ const path = require('node:path');
|
|||||||
// The provider settings ios-preview-controller hands the voice bridge.
|
// The provider settings ios-preview-controller hands the voice bridge.
|
||||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||||
getVoiceTranscriptionLanguage: () => '',
|
getVoiceTranscriptionLanguage: () => '',
|
||||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||||
function load(file, context) {
|
function load(file, context) {
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ test('Android and iOS capture options use the shared language and preserve Trans
|
|||||||
const s = settings();
|
const s = settings();
|
||||||
s.voiceLanguageSetting.set('fr');
|
s.voiceLanguageSetting.set('fr');
|
||||||
s.voiceProviderSetting.set('whisper');
|
s.voiceProviderSetting.set('whisper');
|
||||||
|
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||||
for (const [file, name, method] of [
|
for (const [file, name, method] of [
|
||||||
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
||||||
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
||||||
@@ -75,11 +76,13 @@ test('Android and iOS capture options use the shared language and preserve Trans
|
|||||||
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
||||||
const capture = loadBody(member.body.getText(tree), s, host);
|
const capture = loadBody(member.body.getText(tree), s, host);
|
||||||
assert.equal(capture().language, 'fr');
|
assert.equal(capture().language, 'fr');
|
||||||
|
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||||
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
||||||
assert.equal(capture().provider, 'soniox');
|
assert.equal(capture().provider, 'soniox');
|
||||||
assert.equal(capture().language, 'ja');
|
assert.equal(capture().language, 'ja');
|
||||||
host.continuousClients.clear();
|
host.continuousClients.clear();
|
||||||
assert.equal(capture().language, 'fr');
|
assert.equal(capture().language, 'fr');
|
||||||
|
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -129,3 +132,15 @@ test('multilingual downloads and deletion leave an existing English model intact
|
|||||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
||||||
|
const values = new Map();
|
||||||
|
let s = settings(values);
|
||||||
|
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||||
|
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||||
|
s.voiceLanguageSetting.set('fr');
|
||||||
|
s = settings(values);
|
||||||
|
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
||||||
|
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
||||||
|
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||||
|
});
|
||||||
Reference in new issue
Block a user