forked from Blendman974/faceclaw
Fix EvenHub apps on iOS access to the mic
This commit is contained in:
1 parent
247459bf99
commit
53812609ed
5 files changed
+138
-3
No files matched your search
@@ -16,6 +16,8 @@
|
|||||||
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
|
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
|
||||||
* Fix missing R1 ring battery indicator on iOS
|
* Fix missing R1 ring battery indicator on iOS
|
||||||
* Fix a bug on iOS that would cause corrupted installs of EvenHub apps
|
* Fix a bug on iOS that would cause corrupted installs of EvenHub apps
|
||||||
|
* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable")
|
||||||
|
* EvenHub apps on iOS should now be able to use the mic
|
||||||
|
|
||||||
## 0.8.4 - 2026-10-17
|
## 0.8.4 - 2026-10-17
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
|
|||||||
deliverAudioPcm(pcm: Uint8Array): void;
|
deliverAudioPcm(pcm: Uint8Array): void;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
|
||||||
function activeCommunicator(): any {
|
function activeCommunicator(): any {
|
||||||
if (!global.isAndroid) return null;
|
if (!global.isAndroid) return null;
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
|||||||
* flows (foreground + screen on + no assistant modal).
|
* flows (foreground + screen on + no assistant modal).
|
||||||
*/
|
*/
|
||||||
private audioControl(data: Record<string, unknown>): boolean {
|
private audioControl(data: Record<string, unknown>): boolean {
|
||||||
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
|
|
||||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -443,6 +443,7 @@ export class IosPreviewController {
|
|||||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||||
'Glasses locked. Transcription resumes when you unlock your phone.')
|
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||||
} else this.syncContinuousCapture()
|
} else this.syncContinuousCapture()
|
||||||
|
this.syncRawMicSession()
|
||||||
void (async () => {
|
void (async () => {
|
||||||
await this.displayReady
|
await this.displayReady
|
||||||
const display = this.display
|
const display = this.display
|
||||||
@@ -745,6 +746,8 @@ export class IosPreviewController {
|
|||||||
this.pushBrightness()
|
this.pushBrightness()
|
||||||
this.maybeLaunchOnboarding(this.state)
|
this.maybeLaunchOnboarding(this.state)
|
||||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||||
|
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
|
||||||
|
this.syncRawMicSession()
|
||||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||||
this.syncContinuousSession()
|
this.syncContinuousSession()
|
||||||
this.pushBatteryLevels()
|
this.pushBatteryLevels()
|
||||||
@@ -952,6 +955,8 @@ export class IosPreviewController {
|
|||||||
for (const off of this.sessionOffs.splice(0)) off()
|
for (const off of this.sessionOffs.splice(0)) off()
|
||||||
this.notifications = null
|
this.notifications = null
|
||||||
this.communicator = null
|
this.communicator = null
|
||||||
|
// Releases an EvenHub app's mic before the session closes.
|
||||||
|
this.syncRawMicSession()
|
||||||
this.glance.reset()
|
this.glance.reset()
|
||||||
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
||||||
}
|
}
|
||||||
@@ -1065,6 +1070,15 @@ export class IosPreviewController {
|
|||||||
}
|
}
|
||||||
if (current()) this.syncContinuousClaim()
|
if (current()) this.syncContinuousClaim()
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
|
||||||
|
* unlocked glasses session, as speech capture does; the EvenHub mic router
|
||||||
|
* decides which app hears it and when.
|
||||||
|
*/
|
||||||
|
private syncRawMicSession(): void {
|
||||||
|
const ready = this.state.phase === 'connected' && !this.glassesLocked
|
||||||
|
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
|
||||||
|
}
|
||||||
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||||
private syncContinuousClaim(): void {
|
private syncContinuousClaim(): void {
|
||||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ export type IosMicrophoneSession = {
|
|||||||
}
|
}
|
||||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||||
|
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
|
||||||
|
const RAW_RETRY_MS = 3000
|
||||||
|
const RAW_RETRY_LIMIT = 5
|
||||||
|
|
||||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||||
* from the glasses when connected, or the default phone input in preview mode,
|
* from the glasses when connected, or the default phone input in preview mode,
|
||||||
@@ -53,6 +56,21 @@ export class IosVoiceControlBridge {
|
|||||||
// and it is created by the first audio, so a failed start leaves no file.
|
// and it is created by the first audio, so a failed start leaves no file.
|
||||||
private recordingName = ''
|
private recordingName = ''
|
||||||
private recorder: WavRecorder | null = null
|
private recorder: WavRecorder | null = null
|
||||||
|
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||||
|
// rawListeners while any owner wants it, from the session setRawSession
|
||||||
|
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||||
|
// it; it resumes on its own once that capture has ended.
|
||||||
|
private rawNative: any = null
|
||||||
|
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
|
||||||
|
private readonly rawOwners = new Set<string>()
|
||||||
|
private rawSession: IosMicrophoneSession | null = null
|
||||||
|
// The session the tap enabled the mic on, while it is starting or live.
|
||||||
|
private rawHeld: IosMicrophoneSession | null = null
|
||||||
|
private rawLive = false
|
||||||
|
private rawGeneration = 0
|
||||||
|
private rawFailures = 0
|
||||||
|
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
|
private rawLog: (message: string) => void = () => {}
|
||||||
|
|
||||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||||
@@ -62,7 +80,7 @@ export class IosVoiceControlBridge {
|
|||||||
}
|
}
|
||||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures supply PCM. */
|
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
|
||||||
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
||||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||||
private ensureNative(): any {
|
private ensureNative(): any {
|
||||||
@@ -102,6 +120,8 @@ export class IosVoiceControlBridge {
|
|||||||
get statusText(): string { return this.status }
|
get statusText(): string { return this.status }
|
||||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||||
if (this.joinsContinuous(continuous)) return
|
if (this.joinsContinuous(continuous)) return
|
||||||
|
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
|
||||||
|
this.endRawTap()
|
||||||
this.stop()
|
this.stop()
|
||||||
const generation = ++this.generation
|
const generation = ++this.generation
|
||||||
this.continuous = continuous
|
this.continuous = continuous
|
||||||
@@ -131,6 +151,7 @@ export class IosVoiceControlBridge {
|
|||||||
}
|
}
|
||||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||||
if (this.joinsContinuous(continuous)) return
|
if (this.joinsContinuous(continuous)) return
|
||||||
|
this.endRawTap()
|
||||||
this.stop()
|
this.stop()
|
||||||
this.continuous = continuous
|
this.continuous = continuous
|
||||||
this.log = log; this.source = 'phone'; this.capturing = true
|
this.log = log; this.source = 'phone'; this.capturing = true
|
||||||
@@ -228,6 +249,100 @@ export class IosVoiceControlBridge {
|
|||||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
||||||
this.stop(); this.setStatus(message); this.emitEnd()
|
this.stop(); this.setStatus(message); this.emitEnd()
|
||||||
}
|
}
|
||||||
|
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
|
||||||
|
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
|
||||||
|
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* The glasses session the raw tap captures from (connected and unlocked),
|
||||||
|
* or null. Owners outlive a change: the tap moves to the next session.
|
||||||
|
*/
|
||||||
|
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
|
||||||
|
this.rawLog = log
|
||||||
|
if (session === this.rawSession) return
|
||||||
|
this.endRawTap()
|
||||||
|
this.rawSession = session; this.rawFailures = 0
|
||||||
|
this.syncRawTap()
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Start the decode-only raw tap on behalf of `owner`; the communicator is
|
||||||
|
* the one setRawSession bound, not `options.communicator`. False while
|
||||||
|
* speech capture holds the mic or no session is bound: the owner stays
|
||||||
|
* registered and the tap starts once it can. Idempotent while running.
|
||||||
|
*/
|
||||||
|
startRawCapture(options: { communicator: any; owner: string }): boolean {
|
||||||
|
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
|
||||||
|
this.syncRawTap()
|
||||||
|
return this.rawHeld !== null
|
||||||
|
}
|
||||||
|
/** `owner` no longer wants the raw tap; it stops once no owner does. */
|
||||||
|
stopRawCapture(owner: string): void {
|
||||||
|
if (this.rawOwners.delete(owner)) this.syncRawTap()
|
||||||
|
}
|
||||||
|
/** Whether the raw tap's glasses mic enable went through and still holds. */
|
||||||
|
isRawCaptureLive(): boolean { return this.rawLive }
|
||||||
|
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
|
||||||
|
private syncRawTap(): void {
|
||||||
|
const session = this.rawSession
|
||||||
|
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
|
||||||
|
if (this.rawHeld || this.rawRetryTimer !== null) return
|
||||||
|
void this.beginRawTap(session)
|
||||||
|
}
|
||||||
|
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
|
||||||
|
const generation = ++this.rawGeneration
|
||||||
|
const native = this.ensureRawNative()
|
||||||
|
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
|
||||||
|
if (error) { this.failRawTap(error); return }
|
||||||
|
this.rawHeld = session
|
||||||
|
try {
|
||||||
|
await session.setMicrophone(true, packet => {
|
||||||
|
if (generation !== this.rawGeneration) return
|
||||||
|
const copy = new Uint8Array(packet)
|
||||||
|
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||||
|
})
|
||||||
|
if (generation !== this.rawGeneration) return
|
||||||
|
this.rawLive = true
|
||||||
|
this.rawLog('EvenHub mic: glasses microphone enabled')
|
||||||
|
} catch (failure) {
|
||||||
|
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private ensureRawNative(): any {
|
||||||
|
if (!this.rawNative) {
|
||||||
|
this.rawNative = FaceclawSpeech.new()
|
||||||
|
// A PCM capture only ends on its own when decoding failed or fell behind.
|
||||||
|
this.rawNative.eventHandler = (json: string) => {
|
||||||
|
const event = JSON.parse(json)
|
||||||
|
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
|
||||||
|
}
|
||||||
|
this.rawNative.pcmHandler = (pcm: NSData) => {
|
||||||
|
// Copied: bufferFromData doesn't retain pcm.
|
||||||
|
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||||
|
// Audio is flowing, so a later failure is a new problem worth retrying.
|
||||||
|
this.rawFailures = 0
|
||||||
|
for (const fn of [...this.rawListeners]) fn(bytes)
|
||||||
|
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return this.rawNative
|
||||||
|
}
|
||||||
|
/** Drop the tap and retry it shortly, a bounded number of times. */
|
||||||
|
private failRawTap(message: string): void {
|
||||||
|
this.rawLog(`EvenHub mic: ${message}`)
|
||||||
|
this.endRawTap()
|
||||||
|
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
|
||||||
|
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
|
||||||
|
}
|
||||||
|
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
|
||||||
|
private endRawTap(): void {
|
||||||
|
++this.rawGeneration
|
||||||
|
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
|
||||||
|
this.rawRetryTimer = null
|
||||||
|
this.rawLive = false
|
||||||
|
this.rawNative?.cancel()
|
||||||
|
const session = this.rawHeld; this.rawHeld = null
|
||||||
|
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
|
||||||
|
}
|
||||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||||
private releaseMicrophone(): void {
|
private releaseMicrophone(): void {
|
||||||
const session = this.session; this.session = null
|
const session = this.session; this.session = null
|
||||||
@@ -261,7 +376,11 @@ export class IosVoiceControlBridge {
|
|||||||
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
||||||
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
||||||
}
|
}
|
||||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
private emitEnd(): void {
|
||||||
|
for (const fn of [...this.ends]) fn()
|
||||||
|
// After the listeners: one may start the next capture, which keeps the mic.
|
||||||
|
this.syncRawTap()
|
||||||
|
}
|
||||||
private finishCompletion(): void {
|
private finishCompletion(): void {
|
||||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user