This commit is contained in:
James Babcock committed 2026-10-10 22:56:28 -07:00
commit f6a4615953
24 files changed
+518 -176

No files matched your search

+13 -5
View File
@@ -46,7 +46,9 @@ import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings'
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
@@ -885,7 +887,7 @@ export class IosPreviewController {
if (this.glassesLocked) return false
const usePhoneMic = this.state.phase !== 'connected'
if (usePhoneMic && !this.active) return false
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic)
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
}
@@ -893,14 +895,20 @@ export class IosPreviewController {
if (this.glassesLocked) return
const log = (message: string) => this.logBluetooth(message)
const communicator = this.communicator
const voice = this.voiceProvider()
if (communicator && this.state.phase === 'connected') {
await iosVoiceInput.startGlassesCapture(communicator, log, endpointing)
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
} else if (this.active) {
// Recheck microphone permission if the glasses disconnected after prepare.
if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return
await iosVoiceInput.startPhoneCapture(log, endpointing)
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
}
}
/** The transcription provider setting and the keys the cloud ones need. */
private voiceProvider(): VoiceProviderSelection {
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() }
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
+68
View File
@@ -0,0 +1,68 @@
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control";
/**
* The transcription-provider setting plus the keys the cloud providers need.
* Shared by the Android and iOS voice bridges.
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
};
type CloudProvider = {
name: string;
apiKey: (selection: VoiceProviderSelection) => string;
create: (options: CloudSttOptions) => CloudSttClient;
};
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
elevenlabs: {
name: "ElevenLabs",
apiKey: (selection) => selection.elevenLabsApiKey,
create: (options) => new ElevenLabsSttClient(options),
},
soniox: {
name: "Soniox",
apiKey: (selection) => selection.sonioxApiKey,
create: (options) => new SonioxSttClient(options),
},
whisper: {
name: "OpenAI",
apiKey: (selection) => selection.openAiApiKey,
create: (options) => new OpenAiRealtimeSttClient(options),
},
};
/** Whether createCloudSttClient would return a cloud client for this selection. */
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim());
}
/**
* The cloud provider for a capture, or null to transcribe on-device. A cloud
* provider whose API key is missing falls back to on-device (reported via
* onStatus) rather than failing the capture outright. `continuous` says
* whether the capture is open-ended (Transcribe), where pauses commit segments.
*/
export function createCloudSttClient(
selection: VoiceProviderSelection,
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): CloudSttClient | null {
const provider = CLOUD_PROVIDERS[selection.provider];
if (!provider) return null;
const apiKey = provider.apiKey(selection).trim();
if (!apiKey) {
options.onStatus(`No ${provider.name} key set; using on-device voice.`);
return null;
}
return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous);
}
+9 -3
View File
@@ -2,8 +2,9 @@
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
* Soniox).
* The voice bridge holds one of these while a cloud provider owns the
* transcript; the Java controller then only decodes LC3 to PCM and hands it
* over via acceptPcm.
* transcript; the native capture (FaceclawVoiceController on Android,
* FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over
* via acceptPcm.
*/
export type CloudSttTranscriptEvent = {
@@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any {
return javaBytes;
}
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */
export function encodeBase64(bytes: Uint8Array): string {
if (global.isIOS) {
const copy = bytes.slice();
return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions);
}
if (!global.isAndroid) return "";
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
}
+6 -9
View File
@@ -1,6 +1,5 @@
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
declare const com: any;
import { openSocket, type SocketConnection } from "./socket";
/**
* ElevenLabs realtime speech-to-text over WebSocket.
@@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
export type ElevenLabsSttOptions = CloudSttOptions;
export class ElevenLabsSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// PCM that arrived before the socket finished opening; flushed on open.
@@ -36,7 +34,7 @@ export class ElevenLabsSttClient {
start(): void {
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
this.open = true;
@@ -56,12 +54,12 @@ export class ElevenLabsSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
},
});
};
try {
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
// The API key rides an xi-api-key header on the handshake.
const key = this.options.apiKey;
console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`);
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
this.ws = openSocket(url, listener, { name: "xi-api-key", value: key });
this.options.onStatus("Connecting to ElevenLabs...");
} catch (error) {
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -106,7 +104,6 @@ export class ElevenLabsSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
+5 -13
View File
@@ -1,6 +1,5 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
declare const com: any;
import { openSocket, type SocketConnection } from "./socket";
/**
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
@@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000;
export type OpenAiSttOptions = CloudSttOptions;
export class OpenAiRealtimeSttClient implements CloudSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// Base64 audio (or the commit sentinel) queued until the socket opens.
@@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
start(): void {
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
this.open = true;
@@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
},
});
};
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(
WS_URL,
this.listenerProxy,
"Authorization",
`Bearer ${this.options.apiKey}`,
);
this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` });
this.options.onStatus("Connecting to OpenAI...");
} catch (error) {
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
+14 -4
View File
@@ -1,7 +1,7 @@
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
import type { SocketConnection, SocketHeader, SocketListener } from './socket'
declare const FaceclawSocket: any
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
const native = FaceclawSocket.alloc().initWithURL(url)
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null)
let ended = false
const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() }
const timer = setInterval(() => {
@@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te
else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) }
}
}, 25)
return { sendText: text => { if (!ended) native.sendText(text) }, close }
// NSURLSession queues every send; a failed one surfaces as an error event.
return {
sendText: text => { if (!ended) native.sendText(text); return !ended },
sendBinary: bytes => {
if (ended) return false
const copy = bytes.slice()
native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
return true
},
close,
}
}
+14 -3
View File
@@ -1,9 +1,20 @@
import { toJavaBytes } from './cloud-stt'
declare const com: any
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
/** One extra handshake header, e.g. an API key. */
export type SocketHeader = { name: string; value: string }
export type SocketConnection = {
/** False when the frame was refused outright; later failures arrive via onFailure. */
sendText(text: string): boolean
sendBinary(bytes: Uint8Array): boolean
close(code: number, reason: string): void
}
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener)
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null)
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text),
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null)
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false,
sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false,
close: (code: number, reason: string) => native.close(code, reason) }
return connection
}
+7 -10
View File
@@ -1,8 +1,7 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { openSocket, type SocketConnection } from "./socket";
import { TimedTranscript } from "./transcript-format";
declare const com: any;
/**
* Soniox realtime speech-to-text over WebSocket.
* https://soniox.com/docs/stt/rt/real-time-transcription
@@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5";
export type SonioxSttOptions = CloudSttOptions;
export class SonioxSttClient implements CloudSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// PCM queued until the socket opens and the config message is sent.
@@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient {
start(): void {
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
// The API key rides in the config message; there is no auth header.
@@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
},
});
};
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.ws = openSocket(WS_URL, listener);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingPcm.length = 0;
}
private sendPcm(pcm: Uint8Array): void {
try {
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
if (this.ws?.sendBinary(pcm) === false) {
this.options.onDisconnected?.("Soniox audio send failed.");
}
} catch (error) {
+77 -15
View File
@@ -1,12 +1,17 @@
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
}
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
const CLOUD_FINAL_TIMEOUT_MS = 5000
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode.
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -24,6 +29,12 @@ export class IosVoiceControlBridge {
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: CloudSttClient | null = null
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
@@ -38,16 +49,20 @@ export class IosVoiceControlBridge {
if (!this.native) {
this.native = FaceclawSpeech.new()
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm)))
}
return this.native
}
async prepare(foreground: boolean, usePhoneMic = false): Promise<boolean> {
let status = FaceclawSpeech.authorizationStatus()
if (status === 0) {
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise<boolean> {
// Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio.
if (!usesCloudStt(voice)) {
let status = FaceclawSpeech.authorizationStatus()
if (status === 0) {
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
}
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
}
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
if (usePhoneMic) {
let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus()
if (microphoneStatus === 0) {
@@ -59,12 +74,12 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
this.stop()
const generation = ++this.generation
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
const startError = this.startNative(voice, endpointing, false)
if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return }
try {
await session.setMicrophone(true, packet => {
@@ -74,7 +89,7 @@ export class IosVoiceControlBridge {
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log('Voice: glasses microphone enabled; using on-device recognition')
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
@@ -86,13 +101,30 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise<void> {
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
this.stop()
this.log = log; this.source = 'phone'; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '')
const error = this.startNative(voice, endpointing, true)
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
log('Voice: phone default microphone enabled; using on-device recognition')
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
}
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
const native = this.ensureNative()
const generation = this.generation
const current = () => generation === this.generation
const cloud = createCloudSttClient(voice, {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => false)
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
this.cloud = cloud
cloud.start()
return ''
}
stopPhoneCapture(): void {
if (this.source !== 'phone') return
@@ -111,6 +143,7 @@ export class IosVoiceControlBridge {
this.capturing = this.finalizing = false
this.source = null
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.clearCloud()
this.finishCompletion()
}
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
@@ -122,6 +155,33 @@ export class IosVoiceControlBridge {
const session = this.session; this.session = null
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
private clearCloud(): void {
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
this.cloudTimer = null
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
}
/** The native audio has ended; have the provider finalize what it heard. */
private finishCloud(status: string): void {
this.capturing = false; this.finalizing = true
this.cloudEndStatus = status
this.setStatus('Recognizing…')
// Armed first: finish() can deliver the final synchronously.
this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS)
this.cloud?.finish()
}
private completeCloud(): void {
const status = this.cloudEndStatus ?? 'Ready to send'
this.clearCloud()
this.capturing = this.finalizing = false
this.finishCompletion()
this.setStatus(status); this.emitEnd()
}
private receiveCloudTranscript(event: CloudSttTranscriptEvent): void {
if (!this.capturing && !this.finalizing) return
for (const fn of [...this.transcripts]) fn(event)
// Push-to-talk commits once, so the first final after it is the last.
if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud()
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
@@ -134,13 +194,15 @@ export class IosVoiceControlBridge {
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
else if (event.kind === 'audio') {
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
// A cloud provider's own status (connecting, listening, reconnecting) stays up.
if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
} else if (event.kind === 'finishing') {
void this.stopPushToTalk(); this.emitEnd()
} else if (event.kind === 'ended') {
this.source = null
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone()
if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return }
this.capturing = this.finalizing = false
this.finishCompletion()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
}
+4 -33
View File
@@ -1,11 +1,8 @@
import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
@@ -341,38 +338,12 @@ export class FaceclawVoiceControlBridge {
* than failing the capture outright.
*/
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
const sttOptions = {
apiKey: "",
return createCloudSttClient(options, {
onTranscript: (event: CloudSttTranscriptEvent) =>
this.emitTranscript(event.text, event.isFinal, event),
onStatus: (status: string) => this.setStatus(status),
onError: (message: string) => this.setStatus(message),
};
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
if (options.provider === "elevenlabs") {
const apiKey = options.elevenLabsApiKey.trim();
if (!apiKey) {
this.setStatus("No ElevenLabs key set; using on-device voice.");
return null;
}
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
if (!apiKey) {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return reconnecting((config) => new SonioxSttClient(config), apiKey);
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
return null;
}
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
}, () => this.captureHolders.has("continuous"));
}
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
+7 -3
View File
@@ -803,8 +803,10 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
// below now says "OpenAI" to tell the two apart in the picker.
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
// On iOS "onboard" is Apple's on-device recognizer, and there is no
// "onboard-whisper" (the sherpa-onnx models are Android-only).
const voiceProviderLabels: Record<VoiceProvider, string> = {
onboard: "On-device (Moonshine)",
onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)",
"onboard-whisper": "On-device (Whisper)",
elevenlabs: "ElevenLabs",
whisper: "OpenAI (Whisper)",
@@ -816,7 +818,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
label: "Transcription Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
formatValue: (value) => voiceProviderLabels[value] ?? value,
isDisabled: (value) => {
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
@@ -824,7 +826,9 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
return false;
},
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
description: global.isIOS
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
});
const wakeWordActionLabels: Record<WakeWordAction, string> = {
+3 -4
View File
@@ -233,10 +233,9 @@ function settingsSections(): SettingsSection[] {
const deferred = new Set(["Watch"]);
return sections.map(section => {
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
}] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting)] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
description: `${section.label} integration has not been ported to iOS.`,