Fix EvenHub apps on iOS access to the mic

This commit is contained in:
James Babcock committed 2026-10-25 14:19:45 -07:00
1 parent 247459bf99
commit 53812609ed
5 files changed
+138 -3

No files matched your search

+2
View File
@@ -16,6 +16,8 @@
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks * Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
* Fix missing R1 ring battery indicator on iOS * Fix missing R1 ring battery indicator on iOS
* Fix a bug on iOS that would cause corrupted installs of EvenHub apps * Fix a bug on iOS that would cause corrupted installs of EvenHub apps
* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable")
* EvenHub apps on iOS should now be able to use the mic
## 0.8.4 - 2026-10-17 ## 0.8.4 - 2026-10-17
+1
View File
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
deliverAudioPcm(pcm: Uint8Array): void; deliverAudioPcm(pcm: Uint8Array): void;
}; };
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
function activeCommunicator(): any { function activeCommunicator(): any {
if (!global.isAndroid) return null; if (!global.isAndroid) return null;
try { try {
-1
View File
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
* flows (foreground + screen on + no assistant modal). * flows (foreground + screen on + no assistant modal).
*/ */
private audioControl(data: Record<string, unknown>): boolean { private audioControl(data: Record<string, unknown>): boolean {
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
if (!permissionsIncludeMicrophone(this.manifest.permissions)) { if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
this.log("evenhub: audioControl denied (app declares no microphone permission)"); this.log("evenhub: audioControl denied (app declares no microphone permission)");
return false; return false;
+14
View File
@@ -443,6 +443,7 @@ export class IosPreviewController {
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
'Glasses locked. Transcription resumes when you unlock your phone.') 'Glasses locked. Transcription resumes when you unlock your phone.')
} else this.syncContinuousCapture() } else this.syncContinuousCapture()
this.syncRawMicSession()
void (async () => { void (async () => {
await this.displayReady await this.displayReady
const display = this.display const display = this.display
@@ -745,6 +746,8 @@ export class IosPreviewController {
this.pushBrightness() this.pushBrightness()
this.maybeLaunchOnboarding(this.state) this.maybeLaunchOnboarding(this.state)
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
this.syncRawMicSession()
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
this.syncContinuousSession() this.syncContinuousSession()
this.pushBatteryLevels() this.pushBatteryLevels()
@@ -952,6 +955,8 @@ export class IosPreviewController {
for (const off of this.sessionOffs.splice(0)) off() for (const off of this.sessionOffs.splice(0)) off()
this.notifications = null this.notifications = null
this.communicator = null this.communicator = null
// Releases an EvenHub app's mic before the session closes.
this.syncRawMicSession()
this.glance.reset() this.glance.reset()
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`)) await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
} }
@@ -1065,6 +1070,15 @@ export class IosPreviewController {
} }
if (current()) this.syncContinuousClaim() if (current()) this.syncContinuousClaim()
} }
/**
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
* unlocked glasses session, as speech capture does; the EvenHub mic router
* decides which app hears it and when.
*/
private syncRawMicSession(): void {
const ready = this.state.phase === 'connected' && !this.glassesLocked
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
}
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
private syncContinuousClaim(): void { private syncContinuousClaim(): void {
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
+121 -2
View File
@@ -11,6 +11,9 @@ export type IosMicrophoneSession = {
} }
/** How long a cloud provider gets, after the last audio, to send its final transcript. */ /** How long a cloud provider gets, after the last audio, to send its final transcript. */
const CLOUD_FINAL_TIMEOUT_MS = 5000 const CLOUD_FINAL_TIMEOUT_MS = 5000
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
const RAW_RETRY_MS = 3000
const RAW_RETRY_LIMIT = 5
/** iOS implementation of the shared voice dialog's event bridge. Audio comes /** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode, * from the glasses when connected, or the default phone input in preview mode,
@@ -53,6 +56,21 @@ export class IosVoiceControlBridge {
// and it is created by the first audio, so a failed start leaves no file. // and it is created by the first audio, so a failed start leaves no file.
private recordingName = '' private recordingName = ''
private recorder: WavRecorder | null = null private recorder: WavRecorder | null = null
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
// rawListeners while any owner wants it, from the session setRawSession
// bound. It shares the one glasses mic with speech capture, which preempts
// it; it resumes on its own once that capture has ended.
private rawNative: any = null
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
private readonly rawOwners = new Set<string>()
private rawSession: IosMicrophoneSession | null = null
// The session the tap enabled the mic on, while it is starting or live.
private rawHeld: IosMicrophoneSession | null = null
private rawLive = false
private rawGeneration = 0
private rawFailures = 0
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
private rawLog: (message: string) => void = () => {}
onStatus(listener: (state: VoiceControlState) => void): () => void { onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
@@ -62,7 +80,7 @@ export class IosVoiceControlBridge {
} }
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } } onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} } onSpeechPause(_listener: () => void): () => void { return () => {} }
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */ /** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } } onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) } private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
private ensureNative(): any { private ensureNative(): any {
@@ -102,6 +120,8 @@ export class IosVoiceControlBridge {
get statusText(): string { return this.status } get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> { async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return if (this.joinsContinuous(continuous)) return
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
this.endRawTap()
this.stop() this.stop()
const generation = ++this.generation const generation = ++this.generation
this.continuous = continuous this.continuous = continuous
@@ -131,6 +151,7 @@ export class IosVoiceControlBridge {
} }
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> { async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return if (this.joinsContinuous(continuous)) return
this.endRawTap()
this.stop() this.stop()
this.continuous = continuous this.continuous = continuous
this.log = log; this.source = 'phone'; this.capturing = true this.log = log; this.source = 'phone'; this.capturing = true
@@ -228,6 +249,100 @@ export class IosVoiceControlBridge {
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return } if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
this.stop(); this.setStatus(message); this.emitEnd() this.stop(); this.setStatus(message); this.emitEnd()
} }
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
}
/**
* The glasses session the raw tap captures from (connected and unlocked),
* or null. Owners outlive a change: the tap moves to the next session.
*/
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
this.rawLog = log
if (session === this.rawSession) return
this.endRawTap()
this.rawSession = session; this.rawFailures = 0
this.syncRawTap()
}
/**
* Start the decode-only raw tap on behalf of `owner`; the communicator is
* the one setRawSession bound, not `options.communicator`. False while
* speech capture holds the mic or no session is bound: the owner stays
* registered and the tap starts once it can. Idempotent while running.
*/
startRawCapture(options: { communicator: any; owner: string }): boolean {
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
this.syncRawTap()
return this.rawHeld !== null
}
/** `owner` no longer wants the raw tap; it stops once no owner does. */
stopRawCapture(owner: string): void {
if (this.rawOwners.delete(owner)) this.syncRawTap()
}
/** Whether the raw tap's glasses mic enable went through and still holds. */
isRawCaptureLive(): boolean { return this.rawLive }
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
private syncRawTap(): void {
const session = this.rawSession
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
if (this.rawHeld || this.rawRetryTimer !== null) return
void this.beginRawTap(session)
}
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
const generation = ++this.rawGeneration
const native = this.ensureRawNative()
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
if (error) { this.failRawTap(error); return }
this.rawHeld = session
try {
await session.setMicrophone(true, packet => {
if (generation !== this.rawGeneration) return
const copy = new Uint8Array(packet)
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.rawGeneration) return
this.rawLive = true
this.rawLog('EvenHub mic: glasses microphone enabled')
} catch (failure) {
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
}
}
private ensureRawNative(): any {
if (!this.rawNative) {
this.rawNative = FaceclawSpeech.new()
// A PCM capture only ends on its own when decoding failed or fell behind.
this.rawNative.eventHandler = (json: string) => {
const event = JSON.parse(json)
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
}
this.rawNative.pcmHandler = (pcm: NSData) => {
// Copied: bufferFromData doesn't retain pcm.
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
// Audio is flowing, so a later failure is a new problem worth retrying.
this.rawFailures = 0
for (const fn of [...this.rawListeners]) fn(bytes)
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
}
}
return this.rawNative
}
/** Drop the tap and retry it shortly, a bounded number of times. */
private failRawTap(message: string): void {
this.rawLog(`EvenHub mic: ${message}`)
this.endRawTap()
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
}
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
private endRawTap(): void {
++this.rawGeneration
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
this.rawRetryTimer = null
this.rawLive = false
this.rawNative?.cancel()
const session = this.rawHeld; this.rawHeld = null
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
}
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
private releaseMicrophone(): void { private releaseMicrophone(): void {
const session = this.session; this.session = null const session = this.session; this.session = null
@@ -261,7 +376,11 @@ export class IosVoiceControlBridge {
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive. // In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud() if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
} }
private emitEnd(): void { for (const fn of [...this.ends]) fn() } private emitEnd(): void {
for (const fn of [...this.ends]) fn()
// After the listeners: one may start the next capture, which keeps the mic.
this.syncRawTap()
}
private finishCompletion(): void { private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
} }