forked from Blendman974/faceclaw
iOS TTF support; voice input
This commit is contained in:
1 parent
ae14edcb59
commit
b07ae0b479
67 files changed
+15717
-55
No files matched your search
@@ -21,6 +21,7 @@ const wrap = (command: number, magic: number, field: number, body: Uint8Array) =
|
||||
export const authentication = (magic: number) => wrap(4, magic, 3, concat(integer(1, 1), integer(2, 4)))
|
||||
export const prelude = () => wrap(2, 156, 4, bytes(3, bytes(2, bytes(2, concat(integer(1, 0), integer(2, 0))))))
|
||||
export const heartbeat = (magic: number) => wrap(12, magic, 14, integer(1, 0))
|
||||
export const audioControl = (magic: number, enabled: boolean) => wrap(15, magic, 18, integer(1, enabled ? 1 : 0))
|
||||
export const settingsQuery = (magic: number) => wrap(2, magic, 4, integer(1, 1))
|
||||
export const shutdown = (magic: number) => wrap(9, magic, 11, integer(1, 1))
|
||||
export const framebufferLease = (acquire: boolean) => concat(integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, acquire ? 5 : 6, 0, 0])))
|
||||
|
||||
@@ -51,6 +51,10 @@ export class GlassesSession {
|
||||
private layoutCreated = false
|
||||
private off: () => void
|
||||
private stopping: Promise<void> | null = null
|
||||
private audioListener: ((packet: Uint8Array) => void) | null = null
|
||||
private microphoneToken = 0
|
||||
private microphoneWork: Promise<void> = Promise.resolve()
|
||||
private microphoneEnabled = false
|
||||
constructor(private readonly transport: SessionTransport,
|
||||
private readonly deflate: (data: Uint8Array) => Uint8Array,
|
||||
private readonly onState: (state: SessionState) => void,
|
||||
@@ -211,6 +215,13 @@ export class GlassesSession {
|
||||
return
|
||||
}
|
||||
if (event.kind !== 'notification' || !event.data) return
|
||||
if (event.characteristic === protocol.G2_RENDER_NOTIFY && event.identifier !== this.ids.ring) {
|
||||
if (this.audioListener && this.state.phase === 'connected') {
|
||||
this.audioListener(hexToBytes(event.data))
|
||||
this.onActivity(); this.wake()
|
||||
}
|
||||
return
|
||||
}
|
||||
const ring = event.identifier === this.ids.ring
|
||||
if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return
|
||||
try {
|
||||
@@ -268,6 +279,30 @@ export class GlassesSession {
|
||||
if (this.state.phase !== 'connected') return
|
||||
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
|
||||
}
|
||||
/** Serialize mic enable/disable, including a release while enable awaits ACK. */
|
||||
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void> {
|
||||
const token = ++this.microphoneToken, generation = this.generation
|
||||
this.audioListener = enabled ? listener ?? null : null
|
||||
const current = () => generation === this.generation && token === this.microphoneToken
|
||||
const work = this.microphoneWork.catch(() => {}).then(async () => {
|
||||
if (!current()) return
|
||||
if (this.state.phase !== 'connected') {
|
||||
if (enabled) throw new Error('Connect the glasses before starting voice input.')
|
||||
return
|
||||
}
|
||||
if (enabled) {
|
||||
await Promise.all(['left', 'right'].map(role => this.transport.subscribe(this.ids[role], protocol.G2_RENDER_NOTIFY)))
|
||||
if (!current()) return
|
||||
}
|
||||
await this.request('right', protocol.SID.hub, magic => protocol.audioControl(magic, enabled), enabled ? 'Microphone enable' : 'Microphone disable')
|
||||
if (current()) { this.microphoneEnabled = enabled; this.log(`Glasses microphone ${enabled ? 'enabled' : 'disabled'}`) }
|
||||
}).catch(error => {
|
||||
if (current()) { this.audioListener = null; this.microphoneEnabled = false }
|
||||
throw error
|
||||
})
|
||||
this.microphoneWork = work
|
||||
return work
|
||||
}
|
||||
wake(): void {
|
||||
if (this.timer !== null) { clearTimeout(this.timer); this.timer = null }
|
||||
void this.pump()
|
||||
@@ -308,6 +343,7 @@ export class GlassesSession {
|
||||
}
|
||||
private reset(): void {
|
||||
++this.generation
|
||||
++this.microphoneToken; this.audioListener = null; this.microphoneEnabled = false
|
||||
if (this.timer !== null) clearTimeout(this.timer)
|
||||
if (this.retryTimer !== null) clearTimeout(this.retryTimer)
|
||||
if (this.ringRetryTimer !== null) clearTimeout(this.ringRetryTimer)
|
||||
@@ -337,6 +373,11 @@ export class GlassesSession {
|
||||
}
|
||||
private async stopInternal(): Promise<void> {
|
||||
this.wanted = false
|
||||
// Stop mic capture before ending the display session. Late enable ACKs
|
||||
// cannot leave a microphone streaming after an explicit Disconnect.
|
||||
if (this.audioListener || this.microphoneEnabled) {
|
||||
try { await this.setMicrophone(false) } catch (error) { this.log(`Microphone cleanup: ${this.message(error)}`) }
|
||||
}
|
||||
const cleanup = this.layoutCreated && this.state.phase === 'connected'
|
||||
this.update('disconnecting', 'Disconnecting…'); this.reset()
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescript/core'
|
||||
import { iosBluetooth } from '../native/ios-bluetooth'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { GlassesSession, type SessionState } from './glasses-session'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
@@ -47,7 +48,8 @@ export class IosPreviewController {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: () => this.typeIntoApp(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startTextSettingEdit: setting => this.editSetting(setting),
|
||||
startTextSettingsEdit: async (settings, _title, finished) => {
|
||||
@@ -62,7 +64,8 @@ export class IosPreviewController {
|
||||
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
voiceInputEnabled: false,
|
||||
voiceInputEnabled: true,
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => this.requestShellRender(),
|
||||
@@ -285,7 +288,7 @@ export class IosPreviewController {
|
||||
openSettings: section => { void this.launchApp('settings', { section }) },
|
||||
startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) },
|
||||
endTextSettingEdit: () => {},
|
||||
startTextInput: () => { void this.typeIntoApp() },
|
||||
startTextInput: () => shell.startVoiceInput(),
|
||||
})
|
||||
this.appHosts.set(appId, host)
|
||||
return host
|
||||
@@ -307,6 +310,7 @@ export class IosPreviewController {
|
||||
if (!this.session) {
|
||||
const { deflate } = require('pako') as { deflate: (data: Uint8Array) => Uint8Array }
|
||||
this.session = new GlassesSession(iosBluetooth(), deflate, state => {
|
||||
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState(state)
|
||||
@@ -322,6 +326,20 @@ export class IosPreviewController {
|
||||
await this.session.start(addresses)
|
||||
}
|
||||
async disconnect(): Promise<void> { await this.session?.stop() }
|
||||
startVoiceInput(): void { shell.startVoiceInput() }
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (this.session?.state.phase !== 'connected') {
|
||||
if (this.active) this.onError('Connect the glasses to use their microphone.')
|
||||
return false
|
||||
}
|
||||
const ready = await iosVoiceInput.prepare(this.active)
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && this.session?.state.phase === 'connected'
|
||||
}
|
||||
private async startVoiceCapture(): Promise<void> {
|
||||
if (!this.session || this.session.state.phase !== 'connected') return
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message))
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
||||
|
||||
Reference in new issue
Block a user