Allow choosing an OpenAI transcription model available to the project

This commit is contained in:
Blendman974 committed 2026-11-08 00:35:51 +01:00
1 parent 0b9e4cb367
commit a80d6697b5
11 files changed
+113 -17

No files matched your search

+2 -2
View File
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -2207,7 +2207,7 @@ class DashboardController {
provider,
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(),
+2 -2
View File
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
@@ -1205,7 +1205,7 @@ export class IosPreviewController {
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
const provider = choice?.provider ?? voiceProviderSetting.get()
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get() }
}
+6 -4
View File
@@ -1,6 +1,6 @@
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control";
@@ -19,6 +19,7 @@ export type VoiceProviderSelection = {
language?: string;
elevenLabsApiKey: string;
openAiApiKey: string;
openAiTranscriptionModel?: OpenAiSttModel;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
@@ -52,7 +53,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
whisper: {
name: "OpenAI",
apiKey: (selection) => selection.openAiApiKey,
create: (options) => new OpenAiRealtimeSttClient(options),
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
},
"whisper-server": {
name: "Whisper server",
@@ -76,9 +77,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
* What a capture transcribes with: the provider and the language it is told
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
const fallback = selection.provider === "onboard-whisper" ? "en" : "";
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}`;
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
+10 -7
View File
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
/**
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
* on the Realtime API, with the streaming gpt-realtime-whisper model).
* on the Realtime API, with a configurable transcription model).
* https://developers.openai.com/api/docs/guides/realtime-transcription
*
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
@@ -16,10 +16,11 @@ import { openSocket, type SocketConnection } from "./socket";
*/
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
const MODEL_ID = "gpt-realtime-whisper";
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
const TARGET_SAMPLE_RATE = 24000;
export type OpenAiSttOptions = CloudSttOptions;
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
export class OpenAiRealtimeSttClient implements CloudSttClient {
private ws: SocketConnection | null = null;
@@ -40,7 +41,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
@@ -165,7 +166,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
return;
}
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
this.options.onError(message.error?.code === "model_not_found"
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
return;
default:
return;
@@ -201,7 +204,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
function sessionConfig(language: string): object {
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
return {
type: "session.update",
session: {
@@ -209,7 +212,7 @@ function sessionConfig(language: string): object {
audio: {
input: {
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
transcription: { model, ...(language ? { language } : {}) },
// Push-to-talk owns the utterance boundary; no server VAD.
turn_detection: null,
},
+3 -1
View File
@@ -1,3 +1,4 @@
import type { OpenAiSttModel } from "./openai-stt";
import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
@@ -24,7 +25,7 @@ export type VoiceControlState = {
detail: string;
};
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
@@ -54,6 +55,7 @@ export type PushToTalkOptions = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
openAiApiKey: string;
openAiTranscriptionModel?: OpenAiSttModel;
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
+10
View File
@@ -1,3 +1,4 @@
import type { OpenAiSttModel } from "../native/openai-stt";
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
@@ -1137,6 +1138,15 @@ export const whisperServerModelSetting = new ConfigSettingString({
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
});
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
id: "openai-transcription-model",
label: "OpenAI transcription model",
storageKey: "voice.openAiTranscriptionModel",
defaultValue: "gpt-realtime-whisper",
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
});
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
id: "voice-language",
label: "Transcription language",
+2
View File
@@ -63,6 +63,7 @@ import {
mapboxApiKeySetting,
mirrorTouchSetting,
openAiApiKeySetting,
openAiTranscriptionModelSetting,
previewColorSetting,
previewScalingSetting,
phoneRotationSetting,
@@ -530,6 +531,7 @@ function asrModelStatusText(id: AsrModelId): string {
*/
function whisperServerMenuItems(): MenuItem[] {
return [
enumSettingMenuItem(openAiTranscriptionModelSetting),
textSettingMenuItem(whisperServerUrlSetting),
textSettingMenuItem(whisperServerModelSetting),
textSettingMenuItem(whisperServerLanguageSetting),