iOS TTF support; voice input

This commit is contained in:
James Babcock committed 2026-09-07 16:57:49 -07:00
1 parent ae14edcb59
commit b07ae0b479
67 files changed
+15717 -55

No files matched your search

+1
View File
@@ -21,6 +21,7 @@ const wrap = (command: number, magic: number, field: number, body: Uint8Array) =
export const authentication = (magic: number) => wrap(4, magic, 3, concat(integer(1, 1), integer(2, 4)))
export const prelude = () => wrap(2, 156, 4, bytes(3, bytes(2, bytes(2, concat(integer(1, 0), integer(2, 0))))))
export const heartbeat = (magic: number) => wrap(12, magic, 14, integer(1, 0))
export const audioControl = (magic: number, enabled: boolean) => wrap(15, magic, 18, integer(1, enabled ? 1 : 0))
export const settingsQuery = (magic: number) => wrap(2, magic, 4, integer(1, 1))
export const shutdown = (magic: number) => wrap(9, magic, 11, integer(1, 1))
export const framebufferLease = (acquire: boolean) => concat(integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, acquire ? 5 : 6, 0, 0])))
+41
View File
@@ -51,6 +51,10 @@ export class GlassesSession {
private layoutCreated = false
private off: () => void
private stopping: Promise<void> | null = null
private audioListener: ((packet: Uint8Array) => void) | null = null
private microphoneToken = 0
private microphoneWork: Promise<void> = Promise.resolve()
private microphoneEnabled = false
constructor(private readonly transport: SessionTransport,
private readonly deflate: (data: Uint8Array) => Uint8Array,
private readonly onState: (state: SessionState) => void,
@@ -211,6 +215,13 @@ export class GlassesSession {
return
}
if (event.kind !== 'notification' || !event.data) return
if (event.characteristic === protocol.G2_RENDER_NOTIFY && event.identifier !== this.ids.ring) {
if (this.audioListener && this.state.phase === 'connected') {
this.audioListener(hexToBytes(event.data))
this.onActivity(); this.wake()
}
return
}
const ring = event.identifier === this.ids.ring
if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return
try {
@@ -268,6 +279,30 @@ export class GlassesSession {
if (this.state.phase !== 'connected') return
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
}
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
const token = ++this.microphoneToken, generation = this.generation
this.audioListener = enabled ? listener ?? null : null
const current = () => generation === this.generation && token === this.microphoneToken
const work = this.microphoneWork.catch(() => {}).then(async () => {
if (!current()) return
if (this.state.phase !== 'connected') {
if (enabled) throw new Error('Connect the glasses before starting voice input.')
return
}
if (enabled) {
await Promise.all(['left', 'right'].map(role => this.transport.subscribe(this.ids[role], protocol.G2_RENDER_NOTIFY)))
if (!current()) return
}
await this.request('right', protocol.SID.hub, magic => protocol.audioControl(magic, enabled), enabled ? 'Microphone enable' : 'Microphone disable')
if (current()) { this.microphoneEnabled = enabled; this.log(`Glasses microphone ${enabled ? 'enabled' : 'disabled'}`) }
}).catch(error => {
if (current()) { this.audioListener = null; this.microphoneEnabled = false }
throw error
})
this.microphoneWork = work
return work
}
wake(): void {
if (this.timer !== null) { clearTimeout(this.timer); this.timer = null }
void this.pump()
@@ -308,6 +343,7 @@ export class GlassesSession {
}
private reset(): void {
++this.generation
++this.microphoneToken; this.audioListener = null; this.microphoneEnabled = false
if (this.timer !== null) clearTimeout(this.timer)
if (this.retryTimer !== null) clearTimeout(this.retryTimer)
if (this.ringRetryTimer !== null) clearTimeout(this.ringRetryTimer)
@@ -337,6 +373,11 @@ export class GlassesSession {
}
private async stopInternal(): Promise<void> {
this.wanted = false
// Stop mic capture before ending the display session. Late enable ACKs
// cannot leave a microphone streaming after an explicit Disconnect.
if (this.audioListener || this.microphoneEnabled) {
try { await this.setMicrophone(false) } catch (error) { this.log(`Microphone cleanup: ${this.message(error)}`) }
}
const cleanup = this.layoutCreated && this.state.phase === 'connected'
this.update('disconnecting', 'Disconnecting…'); this.reset()
try {
+21 -3
View File
@@ -1,5 +1,6 @@
import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescript/core'
import { iosBluetooth } from '../native/ios-bluetooth'
import { iosVoiceInput } from '../native/ios-voice-input'
import { GlassesSession, type SessionState } from './glasses-session'
import { loadDeviceAddresses } from './device-addresses'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -47,7 +48,8 @@ export class IosPreviewController {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
disconnect: () => this.disconnect(),
startVoiceCapture: () => this.typeIntoApp(),
startVoiceCapture: () => this.startVoiceCapture(),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
startTextSettingEdit: setting => this.editSetting(setting),
startTextSettingsEdit: async (settings, _title, finished) => {
@@ -62,7 +64,8 @@ export class IosPreviewController {
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
shell.configure({
actions: this.actions,
voiceInputEnabled: false,
voiceInputEnabled: true,
prepareVoiceCapture: () => this.prepareVoiceCapture(),
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
@@ -285,7 +288,7 @@ export class IosPreviewController {
openSettings: section => { void this.launchApp('settings', { section }) },
startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) },
endTextSettingEdit: () => {},
startTextInput: () => { void this.typeIntoApp() },
startTextInput: () => shell.startVoiceInput(),
})
this.appHosts.set(appId, host)
return host
@@ -307,6 +310,7 @@ export class IosPreviewController {
if (!this.session) {
const { deflate } = require('pako') as { deflate: (data: Uint8Array) => Uint8Array }
this.session = new GlassesSession(iosBluetooth(), deflate, state => {
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
this.syncRuntime()
if (this.active) this.onConnectionState(state)
@@ -322,6 +326,20 @@ export class IosPreviewController {
await this.session.start(addresses)
}
async disconnect(): Promise<void> { await this.session?.stop() }
startVoiceInput(): void { shell.startVoiceInput() }
private async prepareVoiceCapture(): Promise<boolean> {
if (this.session?.state.phase !== 'connected') {
if (this.active) this.onError('Connect the glasses to use their microphone.')
return false
}
const ready = await iosVoiceInput.prepare(this.active)
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && this.session?.state.phase === 'connected'
}
private async startVoiceCapture(): Promise<void> {
if (!this.session || this.session.state.phase !== 'connected') return
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
console.log(`[ios-ble] ${line}`); this.logLines.push(line)