forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw
This commit is contained in:
commit
f6a4615953
24 files changed
+518
-176
No files matched your search
@@ -46,7 +46,9 @@ import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings'
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
@@ -885,7 +887,7 @@ export class IosPreviewController {
|
||||
if (this.glassesLocked) return false
|
||||
const usePhoneMic = this.state.phase !== 'connected'
|
||||
if (usePhoneMic && !this.active) return false
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic)
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
|
||||
}
|
||||
@@ -893,14 +895,20 @@ export class IosPreviewController {
|
||||
if (this.glassesLocked) return
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
const communicator = this.communicator
|
||||
const voice = this.voiceProvider()
|
||||
if (communicator && this.state.phase === 'connected') {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, log, endpointing)
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
|
||||
} else if (this.active) {
|
||||
// Recheck microphone permission if the glasses disconnected after prepare.
|
||||
if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(log, endpointing)
|
||||
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||
}
|
||||
}
|
||||
/** The transcription provider setting and the keys the cloud ones need. */
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
return { provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
|
||||
/**
|
||||
* The transcription-provider setting plus the keys the cloud providers need.
|
||||
* Shared by the Android and iOS voice bridges.
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
};
|
||||
|
||||
type CloudProvider = {
|
||||
name: string;
|
||||
apiKey: (selection: VoiceProviderSelection) => string;
|
||||
create: (options: CloudSttOptions) => CloudSttClient;
|
||||
};
|
||||
|
||||
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
|
||||
const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
elevenlabs: {
|
||||
name: "ElevenLabs",
|
||||
apiKey: (selection) => selection.elevenLabsApiKey,
|
||||
create: (options) => new ElevenLabsSttClient(options),
|
||||
},
|
||||
soniox: {
|
||||
name: "Soniox",
|
||||
apiKey: (selection) => selection.sonioxApiKey,
|
||||
create: (options) => new SonioxSttClient(options),
|
||||
},
|
||||
whisper: {
|
||||
name: "OpenAI",
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
},
|
||||
};
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
|
||||
return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* The cloud provider for a capture, or null to transcribe on-device. A cloud
|
||||
* provider whose API key is missing falls back to on-device (reported via
|
||||
* onStatus) rather than failing the capture outright. `continuous` says
|
||||
* whether the capture is open-ended (Transcribe), where pauses commit segments.
|
||||
*/
|
||||
export function createCloudSttClient(
|
||||
selection: VoiceProviderSelection,
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): CloudSttClient | null {
|
||||
const provider = CLOUD_PROVIDERS[selection.provider];
|
||||
if (!provider) return null;
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
if (!apiKey) {
|
||||
options.onStatus(`No ${provider.name} key set; using on-device voice.`);
|
||||
return null;
|
||||
}
|
||||
return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous);
|
||||
}
|
||||
@@ -2,8 +2,9 @@
|
||||
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
|
||||
* Soniox).
|
||||
* The voice bridge holds one of these while a cloud provider owns the
|
||||
* transcript; the Java controller then only decodes LC3 to PCM and hands it
|
||||
* over via acceptPcm.
|
||||
* transcript; the native capture (FaceclawVoiceController on Android,
|
||||
* FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over
|
||||
* via acceptPcm.
|
||||
*/
|
||||
|
||||
export type CloudSttTranscriptEvent = {
|
||||
@@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any {
|
||||
return javaBytes;
|
||||
}
|
||||
|
||||
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
|
||||
/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */
|
||||
export function encodeBase64(bytes: Uint8Array): string {
|
||||
if (global.isIOS) {
|
||||
const copy = bytes.slice();
|
||||
return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||
.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions);
|
||||
}
|
||||
if (!global.isAndroid) return "";
|
||||
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* ElevenLabs realtime speech-to-text over WebSocket.
|
||||
@@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
|
||||
export type ElevenLabsSttOptions = CloudSttOptions;
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
@@ -36,7 +34,7 @@ export class ElevenLabsSttClient {
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
@@ -56,12 +54,12 @@ export class ElevenLabsSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
|
||||
// The API key rides an xi-api-key header on the handshake.
|
||||
const key = this.options.apiKey;
|
||||
console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`);
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
|
||||
this.ws = openSocket(url, listener, { name: "xi-api-key", value: key });
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -106,7 +104,6 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
@@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000;
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
@@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
@@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(
|
||||
WS_URL,
|
||||
this.listenerProxy,
|
||||
"Authorization",
|
||||
`Bearer ${this.options.apiKey}`,
|
||||
);
|
||||
this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` });
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
|
||||
import type { SocketConnection, SocketHeader, SocketListener } from './socket'
|
||||
declare const FaceclawSocket: any
|
||||
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
|
||||
const native = FaceclawSocket.alloc().initWithURL(url)
|
||||
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
|
||||
const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null)
|
||||
let ended = false
|
||||
const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() }
|
||||
const timer = setInterval(() => {
|
||||
@@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te
|
||||
else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) }
|
||||
}
|
||||
}, 25)
|
||||
return { sendText: text => { if (!ended) native.sendText(text) }, close }
|
||||
// NSURLSession queues every send; a failed one surfaces as an error event.
|
||||
return {
|
||||
sendText: text => { if (!ended) native.sendText(text); return !ended },
|
||||
sendBinary: bytes => {
|
||||
if (ended) return false
|
||||
const copy = bytes.slice()
|
||||
native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
return true
|
||||
},
|
||||
close,
|
||||
}
|
||||
}
|
||||
+14
-3
@@ -1,9 +1,20 @@
|
||||
import { toJavaBytes } from './cloud-stt'
|
||||
|
||||
declare const com: any
|
||||
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
|
||||
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
|
||||
/** One extra handshake header, e.g. an API key. */
|
||||
export type SocketHeader = { name: string; value: string }
|
||||
export type SocketConnection = {
|
||||
/** False when the frame was refused outright; later failures arrive via onFailure. */
|
||||
sendText(text: string): boolean
|
||||
sendBinary(bytes: Uint8Array): boolean
|
||||
close(code: number, reason: string): void
|
||||
}
|
||||
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
|
||||
const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener)
|
||||
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null)
|
||||
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text),
|
||||
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null)
|
||||
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false,
|
||||
sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false,
|
||||
close: (code: number, reason: string) => native.close(code, reason) }
|
||||
return connection
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { openSocket, type SocketConnection } from "./socket";
|
||||
import { TimedTranscript } from "./transcript-format";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Soniox realtime speech-to-text over WebSocket.
|
||||
* https://soniox.com/docs/stt/rt/real-time-transcription
|
||||
@@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5";
|
||||
export type SonioxSttOptions = CloudSttOptions;
|
||||
|
||||
export class SonioxSttClient implements CloudSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private ws: SocketConnection | null = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
@@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
@@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
if (this.closed) return;
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
};
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.ws = openSocket(WS_URL, listener);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
@@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingPcm.length = 0;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
|
||||
if (this.ws?.sendBinary(pcm) === false) {
|
||||
this.options.onDisconnected?.("Soniox audio send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
|
||||
}
|
||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode.
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
@@ -24,6 +29,12 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: CloudSttClient | null = null
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -38,16 +49,20 @@ export class IosVoiceControlBridge {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm)))
|
||||
}
|
||||
return this.native
|
||||
}
|
||||
async prepare(foreground: boolean, usePhoneMic = false): Promise<boolean> {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise<boolean> {
|
||||
// Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio.
|
||||
if (!usesCloudStt(voice)) {
|
||||
let status = FaceclawSpeech.authorizationStatus()
|
||||
if (status === 0) {
|
||||
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
|
||||
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
}
|
||||
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
|
||||
if (usePhoneMic) {
|
||||
let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus()
|
||||
if (microphoneStatus === 0) {
|
||||
@@ -59,12 +74,12 @@ export class IosVoiceControlBridge {
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
|
||||
const startError = this.startNative(voice, endpointing, false)
|
||||
if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
@@ -74,7 +89,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log('Voice: glasses microphone enabled; using on-device recognition')
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -86,13 +101,30 @@ export class IosVoiceControlBridge {
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '')
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log('Voice: phone default microphone enabled; using on-device recognition')
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => false)
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
this.cloud = cloud
|
||||
cloud.start()
|
||||
return ''
|
||||
}
|
||||
stopPhoneCapture(): void {
|
||||
if (this.source !== 'phone') return
|
||||
@@ -111,6 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.capturing = this.finalizing = false
|
||||
this.source = null
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.clearCloud()
|
||||
this.finishCompletion()
|
||||
}
|
||||
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
|
||||
@@ -122,6 +155,33 @@ export class IosVoiceControlBridge {
|
||||
const session = this.session; this.session = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private clearCloud(): void {
|
||||
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
|
||||
this.cloudTimer = null
|
||||
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
|
||||
}
|
||||
/** The native audio has ended; have the provider finalize what it heard. */
|
||||
private finishCloud(status: string): void {
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.cloudEndStatus = status
|
||||
this.setStatus('Recognizing…')
|
||||
// Armed first: finish() can deliver the final synchronously.
|
||||
this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS)
|
||||
this.cloud?.finish()
|
||||
}
|
||||
private completeCloud(): void {
|
||||
const status = this.cloudEndStatus ?? 'Ready to send'
|
||||
this.clearCloud()
|
||||
this.capturing = this.finalizing = false
|
||||
this.finishCompletion()
|
||||
this.setStatus(status); this.emitEnd()
|
||||
}
|
||||
private receiveCloudTranscript(event: CloudSttTranscriptEvent): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
for (const fn of [...this.transcripts]) fn(event)
|
||||
// Push-to-talk commits once, so the first final after it is the last.
|
||||
if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
@@ -134,13 +194,15 @@ export class IosVoiceControlBridge {
|
||||
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
|
||||
else if (event.kind === 'audio') {
|
||||
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
|
||||
if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
|
||||
// A cloud provider's own status (connecting, listening, reconnecting) stays up.
|
||||
if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
|
||||
} else if (event.kind === 'finishing') {
|
||||
void this.stopPushToTalk(); this.emitEnd()
|
||||
} else if (event.kind === 'ended') {
|
||||
this.source = null
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return }
|
||||
this.capturing = this.finalizing = false
|
||||
this.finishCompletion()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
@@ -341,38 +338,12 @@ export class FaceclawVoiceControlBridge {
|
||||
* than failing the capture outright.
|
||||
*/
|
||||
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
|
||||
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
return createCloudSttClient(options, {
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
this.emitTranscript(event.text, event.isFinal, event),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
|
||||
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new SonioxSttClient(config), apiKey);
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
|
||||
}, () => this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
|
||||
@@ -803,8 +803,10 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device (Moonshine)",
|
||||
onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
@@ -816,7 +818,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox"] : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -824,7 +826,9 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
|
||||
@@ -233,10 +233,9 @@ function settingsSections(): SettingsSection[] {
|
||||
const deferred = new Set(["Watch"]);
|
||||
return sections.map(section => {
|
||||
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
|
||||
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
|
||||
}] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting)] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
|
||||
Reference in new issue
Block a user