forked from Blendman974/faceclaw
iOS wakeword support
This commit is contained in:
1 parent
d1df4bf046
commit
15a3e129b2
17 files changed
+341
-137
No files matched your search
@@ -52,7 +52,7 @@ export const readString = (data: Uint8Array, field: number): string => protocol.
|
||||
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
|
||||
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
|
||||
}
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
function input(event: any): GlassesInput | null {
|
||||
return event ? { kind: event.kind === 'sys-event' && event.eventType === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : event.kind,
|
||||
eventType: event.eventType, eventSource: event.eventSource, systemExitReasonCode: event.systemExitReasonCode,
|
||||
|
||||
@@ -127,7 +127,7 @@ export function readString(data: Uint8Array, number: number): string { return Ar
|
||||
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
|
||||
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
|
||||
}
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
|
||||
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
|
||||
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
|
||||
const events = readBytes(message.payload, 13); if (!events) return null
|
||||
|
||||
@@ -93,7 +93,7 @@ export class IosPreviewController {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
@@ -437,9 +437,9 @@ export class IosPreviewController {
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && this.session?.state.phase === 'connected'
|
||||
}
|
||||
private async startVoiceCapture(): Promise<void> {
|
||||
private async startVoiceCapture(endpointing = false): Promise<void> {
|
||||
if (!this.session || this.session.state.phase !== 'connected') return
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing)
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
@@ -18,6 +18,8 @@ export class IosVoiceControlBridge {
|
||||
private session: IosMicrophoneSession | null = null
|
||||
private capturing = false
|
||||
private finalizing = false
|
||||
private completion: Promise<void> = Promise.resolve()
|
||||
private resolveCompletion: (() => void) | null = null
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
@@ -48,12 +50,13 @@ export class IosVoiceControlBridge {
|
||||
return true
|
||||
}
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
|
||||
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.log = log; this.session = session; this.packets = 0; this.capturing = true
|
||||
const error = String(this.ensureNative().start() ?? '')
|
||||
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
@@ -74,16 +77,19 @@ export class IosVoiceControlBridge {
|
||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||
}
|
||||
}
|
||||
stopPushToTalk(): void {
|
||||
if (!this.capturing) return
|
||||
stopPushToTalk(): Promise<void> {
|
||||
const completion = this.completion
|
||||
if (!this.capturing) return completion
|
||||
this.capturing = false; this.finalizing = true
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.setStatus('Recognizing…'); this.native?.finish()
|
||||
return completion
|
||||
}
|
||||
stop(): void {
|
||||
++this.generation
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.finishCompletion()
|
||||
}
|
||||
handleSessionEnded(): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
@@ -95,6 +101,9 @@ export class IosVoiceControlBridge {
|
||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
}
|
||||
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
if (event.kind === 'transcript') {
|
||||
@@ -109,6 +118,7 @@ export class IosVoiceControlBridge {
|
||||
} else if (event.kind === 'ended') {
|
||||
this.capturing = this.finalizing = false
|
||||
this.clearAudioTimer(); this.releaseMicrophone()
|
||||
this.finishCompletion()
|
||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +130,10 @@ function settingsSections(): SettingsSection[] {
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
// iOS has no local LLM, wakeword event delivery, or agent bridge yet.
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
textSettingMenuItem(assistantBridgeTokenSetting),
|
||||
@@ -245,9 +245,9 @@ function settingsSections(): SettingsSection[] {
|
||||
const deferred = new Set(["Navigate", "Watch"]);
|
||||
return sections.map(section => {
|
||||
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
if (section.label === "Voice") return { label: "Voice", items: [{
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
|
||||
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
|
||||
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
|
||||
}] };
|
||||
if (deferred.has(section.label)) return { label: section.label, items: [{
|
||||
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
|
||||
|
||||
Reference in new issue
Block a user