iOS wakeword support

This commit is contained in:
James Babcock committed 2026-09-19 01:37:36 -07:00
1 parent d1df4bf046
commit 15a3e129b2
17 files changed
+341 -137

No files matched your search

+1 -1
View File
@@ -52,7 +52,7 @@ export const readString = (data: Uint8Array, field: number): string => protocol.
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
function input(event: any): GlassesInput | null {
return event ? { kind: event.kind === 'sys-event' && event.eventType === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : event.kind,
eventType: event.eventType, eventSource: event.eventSource, systemExitReasonCode: event.systemExitReasonCode,
+1 -1
View File
@@ -127,7 +127,7 @@ export function readString(data: Uint8Array, number: number): string { return Ar
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake' | 'even-ai'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
const events = readBytes(message.payload, 13); if (!events) return null
+3 -3
View File
@@ -93,7 +93,7 @@ export class IosPreviewController {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
disconnect: () => this.disconnect(),
startVoiceCapture: () => this.startVoiceCapture(),
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
startTextSettingEdit: async setting => { await this.editSetting(setting) },
@@ -437,9 +437,9 @@ export class IosPreviewController {
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && this.session?.state.phase === 'connected'
}
private async startVoiceCapture(): Promise<void> {
private async startVoiceCapture(endpointing = false): Promise<void> {
if (!this.session || this.session.state.phase !== 'connected') return
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing)
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
+15 -5
View File
@@ -18,6 +18,8 @@ export class IosVoiceControlBridge {
private session: IosMicrophoneSession | null = null
private capturing = false
private finalizing = false
private completion: Promise<void> = Promise.resolve()
private resolveCompletion: (() => void) | null = null
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
@@ -48,12 +50,13 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
this.stop()
const generation = ++this.generation
this.log = log; this.session = session; this.packets = 0; this.capturing = true
const error = String(this.ensureNative().start() ?? '')
if (error) { this.capturing = false; this.session = null; this.setStatus(error); this.emitEnd(); return }
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
try {
await session.setMicrophone(true, packet => {
if (generation !== this.generation || !this.capturing) return
@@ -74,16 +77,19 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
stopPushToTalk(): void {
if (!this.capturing) return
stopPushToTalk(): Promise<void> {
const completion = this.completion
if (!this.capturing) return completion
this.capturing = false; this.finalizing = true
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus('Recognizing…'); this.native?.finish()
return completion
}
stop(): void {
++this.generation
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.finishCompletion()
}
handleSessionEnded(): void {
if (!this.capturing && !this.finalizing) return
@@ -95,6 +101,9 @@ export class IosVoiceControlBridge {
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
}
private receive(event: { kind: string; text?: string; final?: boolean; message?: string; seconds?: number; packets?: number; rms?: number; missing?: number; errors?: number }): void {
if (!this.capturing && !this.finalizing) return
if (event.kind === 'transcript') {
@@ -109,6 +118,7 @@ export class IosVoiceControlBridge {
} else if (event.kind === 'ended') {
this.capturing = this.finalizing = false
this.clearAudioTimer(); this.releaseMicrophone()
this.finishCompletion()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
}
}
+5 -5
View File
@@ -130,10 +130,10 @@ function settingsSections(): SettingsSection[] {
// iOS offers cloud APIs; Android also supports local models and the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
// iOS has no local LLM, wakeword event delivery, or agent bridge yet.
...(!global.isIOS ? [localModelMenuItem()] : []),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
...(!global.isIOS ? [
localModelMenuItem(),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
textSettingMenuItem(assistantBridgeTokenSetting),
@@ -245,9 +245,9 @@ function settingsSections(): SettingsSection[] {
const deferred = new Set(["Navigate", "Watch"]);
return sections.map(section => {
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
if (section.label === "Voice") return { label: "Voice", items: [{
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), {
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",
description: "Uses the glasses microphone and your iPhone's speech language. No transcription API key needed. Say Hey Even for hands-free input, or open Voice from the menu and click when finished.",
}] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},