forked from Blendman974/faceclaw
[verified] merge upstream 25cb255b into all-features
This commit is contained in:
commit
ffcdae660d
169 files changed
+16201
-597
No files matched your search
@@ -3,11 +3,18 @@ import type { DiscoveryEvents } from './device-discovery'
|
||||
import type { RawAdvertisement } from '../g2/even-advertisement'
|
||||
export { buildAddressSet } from './device-discovery-common'
|
||||
|
||||
/**
|
||||
* How often a scan re-reads which arms the system is connected to. Each read refreshes
|
||||
* their rows, which age out like advertisements once the link drops and they go quiet.
|
||||
*/
|
||||
const CONNECTED_ARMS_POLL_MS = 2000
|
||||
|
||||
export class DeviceDiscoveryBridge {
|
||||
private ble = iosBluetooth()
|
||||
private off: (() => void) | null = null
|
||||
private generation = 0
|
||||
private cancelCollection: (() => void) | null = null
|
||||
private connectedArmsTimer: ReturnType<typeof setInterval> | null = null
|
||||
get bluetoothEnabled(): boolean { return this.ble.state === 5 }
|
||||
get scanning(): boolean { return !!this.off && this.ble.scanning }
|
||||
startScan(events: DiscoveryEvents): boolean {
|
||||
@@ -21,16 +28,27 @@ export class DeviceDiscoveryBridge {
|
||||
void this.ble.startScan(24 * 60 * 60 * 1000).catch(error => {
|
||||
if (generation === this.generation) events.onScanFailed?.(-1, String(error.message ?? error))
|
||||
})
|
||||
// A connected arm doesn't advertise (iOS holds the right arm's ANCS link open after a
|
||||
// session ends), so the scan alone would show those glasses one-armed.
|
||||
const reportConnectedArms = () => { for (const device of this.ble.systemConnectedDevices()) events.onAdvertisement(device) }
|
||||
reportConnectedArms()
|
||||
this.connectedArmsTimer = setInterval(reportConnectedArms, CONNECTED_ARMS_POLL_MS)
|
||||
return true
|
||||
}
|
||||
stopScan(): void {
|
||||
++this.generation; this.cancelCollection?.()
|
||||
if (this.connectedArmsTimer) { clearInterval(this.connectedArmsTimer); this.connectedArmsTimer = null }
|
||||
if (this.off) { this.off(); this.off = null; this.ble.stopScan() }
|
||||
}
|
||||
// CoreBluetooth does not expose the phone's bond list. Fresh advertisements
|
||||
// supply both identity and the UUID mapping that iOS needs for connections.
|
||||
// supply both identity and the UUID mapping that iOS needs for connections;
|
||||
// startScan() reports the arms the system is connected to.
|
||||
emitBondedDevices(): void {}
|
||||
async getBondedCandidates(): Promise<RawAdvertisement[]> { return [...this.ble.devices.values()] }
|
||||
async getBondedCandidates(): Promise<RawAdvertisement[]> {
|
||||
const devices = new Map(this.ble.devices)
|
||||
for (const device of this.ble.systemConnectedDevices()) if (!devices.has(device.address)) devices.set(device.address, device)
|
||||
return [...devices.values()]
|
||||
}
|
||||
async scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
|
||||
const generation = this.generation
|
||||
await this.ble.ensureReady()
|
||||
|
||||
@@ -19,6 +19,7 @@ declare const FaceclawKitFaceclawCompassListener: any;
|
||||
declare const FaceclawKitFaceclawImuListener: any;
|
||||
declare const FaceclawKitIosAmbientLightListener: any;
|
||||
declare const FaceclawKitIosMicStatusListener: any;
|
||||
declare const FaceclawKitIosKeyboardEventListener: any;
|
||||
|
||||
|
||||
export type CommunicatorPhase =
|
||||
@@ -198,6 +199,13 @@ class MicStatusListener extends NSObject {
|
||||
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class KeyboardEventListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosKeyboardEventListener];
|
||||
handler!: (body: Uint8Array) => void;
|
||||
onNsDataKeyboardEventBody(body: NSData): void { this.handler(fromData(body)); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS twin of the Android communicator bridge: the same public API over
|
||||
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
|
||||
@@ -803,6 +811,19 @@ export class FaceclawCommunicatorBridge {
|
||||
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
|
||||
}
|
||||
|
||||
/** Raw field-131 keyboard-bridge records, in arrival order, on the main queue. */
|
||||
addKeyboardEventListener(listener: (body: Uint8Array) => void): () => void {
|
||||
const proxy = KeyboardEventListener.new() as KeyboardEventListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addKeyboardEventListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeKeyboardEventListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Queue a CFW keyboard-bridge control for the right temple; false when nothing was queued. */
|
||||
sendFaceclawKeyboardControl(record: Uint8Array, label: string): boolean {
|
||||
return Boolean(this.communicator.sendFaceclawKeyboardControlRecordLabel(toData(new Uint8Array(record)), label));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every audio-characteristic packet, whatever enabled the mics, on the
|
||||
* main queue. Passive: sends nothing and doesn't affect capture.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { openSocket } from "./socket";
|
||||
import { openSocket, type SocketConnection, type SocketListener } from "./socket";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState } from "./terminal-session-types";
|
||||
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
@@ -19,54 +20,7 @@ const LAUNCH_REPLY_TIMEOUT_MS = 10_000;
|
||||
// otherwise treat the "\r" as a pasted newline instead of the Enter key.
|
||||
const SUBMIT_DELAY_MS = 150;
|
||||
|
||||
export type G2MirrorSession = {
|
||||
socket: string;
|
||||
pid: number;
|
||||
cwdHint: string;
|
||||
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
|
||||
lastBellAt: number | null;
|
||||
/**
|
||||
* Unix epoch ms (host clock) of the app's last output, or null if none
|
||||
* observed. Fresh to within the wrapper's 2s activity reporting interval
|
||||
* while we're connected; compare against other host timestamps, not the
|
||||
* phone clock (see the worker's local-receive-time activity tracking).
|
||||
*/
|
||||
lastOutputAt: number | null;
|
||||
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
|
||||
title: string | null;
|
||||
};
|
||||
|
||||
export type G2MirrorPhase =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "connected" // handshake accepted; can list/attach
|
||||
| "attached" // relaying to one session
|
||||
| "failed";
|
||||
|
||||
export type G2MirrorState = {
|
||||
phase: G2MirrorPhase;
|
||||
status: string;
|
||||
sessions: G2MirrorSession[];
|
||||
attachedCommand: string;
|
||||
/**
|
||||
* server_name from the init success reply: a human-readable name for the
|
||||
* machine the server runs on. "" until the handshake succeeds (or when an
|
||||
* older server omits the field).
|
||||
*/
|
||||
serverName: string;
|
||||
};
|
||||
|
||||
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
|
||||
export type G2MirrorHistoryReply = {
|
||||
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
|
||||
start: number;
|
||||
/** Oldest index still retained by the wrapper (older requests return nothing). */
|
||||
oldest: number;
|
||||
/** Current splice point (next line to be archived). */
|
||||
next: number;
|
||||
/** Plain text, ANSI-stripped, oldest-to-newest. */
|
||||
lines: string[];
|
||||
};
|
||||
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
|
||||
|
||||
export type G2MirrorClientOptions = {
|
||||
/** TLS (g2mirrors:// → wss) vs plain (g2mirror:// → ws). */
|
||||
@@ -77,6 +31,12 @@ export type G2MirrorClientOptions = {
|
||||
deviceName: string;
|
||||
cols: number;
|
||||
rows: number;
|
||||
/**
|
||||
* Opens the websocket some other way than dialing host:port directly
|
||||
* (e.g. through an SSH tunnel); `label` then names the endpoint in status
|
||||
* lines instead of host:port.
|
||||
*/
|
||||
transport?: { open: (listener: SocketListener) => SocketConnection; label: string };
|
||||
};
|
||||
|
||||
type TerminalDataKind = "snapshot" | "output";
|
||||
@@ -180,7 +140,8 @@ export class G2MirrorClient {
|
||||
this.stopped = false;
|
||||
const generation = ++this.connectionGeneration;
|
||||
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
|
||||
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
|
||||
const transport = this.options.transport;
|
||||
this.setState("connecting", `Connecting to ${transport ? transport.label : `${this.options.host}:${this.options.port}`}...`);
|
||||
this.listenerProxy = {
|
||||
onOpen: () => {
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
@@ -208,7 +169,7 @@ export class G2MirrorClient {
|
||||
},
|
||||
};
|
||||
try {
|
||||
this.ws = openSocket(url, this.listenerProxy);
|
||||
this.ws = transport ? transport.open(this.listenerProxy!) : openSocket(url, this.listenerProxy);
|
||||
} catch (error) {
|
||||
this.ws = null;
|
||||
this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Head-up (look-up wake) trigger on the active glasses session
|
||||
* (GlassesSessionCore.setHeadUpTrigger / getHeadUpStatus). Android only for
|
||||
* now (headUpControlsAvailable): on iOS, or with no session, the setter does
|
||||
* nothing and the getter returns null.
|
||||
*/
|
||||
|
||||
/** Whether this platform can change the glasses' head-up setting at all. */
|
||||
export function headUpControlsAvailable(): boolean {
|
||||
return Boolean(global.isAndroid);
|
||||
}
|
||||
|
||||
/** GlassesSessionCore.getHeadUpStatus. switch -1 = not read on this connection. */
|
||||
export type HeadUpStatus = {
|
||||
switch: number;
|
||||
angle: number;
|
||||
calibration: number;
|
||||
/** The trigger Faceclaw is maintaining, or -1 when it leaves the glasses' own setting. */
|
||||
desired: number;
|
||||
readAgeMs: number;
|
||||
/** The glasses' own switch and angle from before Faceclaw changed them (switch -1 = none held). */
|
||||
originalSwitch: number;
|
||||
originalAngle: number;
|
||||
/** Those originals are being written back. */
|
||||
restorePending: boolean;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the session to keep the head-up trigger at `degrees` (1..90); 0 stops
|
||||
* managing it and writes back `original` (or what the session saved itself)
|
||||
* if Faceclaw changed the glasses' values.
|
||||
*/
|
||||
export function setHeadUpTrigger(degrees: number, original: { switch: number; angle: number } | null = null): boolean {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return false;
|
||||
try {
|
||||
active.setHeadUpTrigger(Math.round(degrees), original ? original.switch : -1, original ? Math.round(original.angle) : 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(`setHeadUpTrigger failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function getHeadUpStatus(): HeadUpStatus | null {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return null;
|
||||
try {
|
||||
return JSON.parse(String(active.getHeadUpStatus())) as HeadUpStatus;
|
||||
} catch (error) {
|
||||
console.warn(`getHeadUpStatus failed: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,23 @@
|
||||
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
|
||||
|
||||
/**
|
||||
* An accelerometer sample in g, in Faceclaw's frame for every glasses model
|
||||
* (the G2's own; the session rotates other models' samples into it, see
|
||||
* ImuOrientation.kt): +x forward, +y to the wearer's left, +z up through the
|
||||
* top of the head. At rest it is the "up" direction: head level reads
|
||||
* (0, 0, 1), looking 45° up (0.71, 0, 0.71), the head tilted 45° to the right
|
||||
* (0, 0.71, 0.71).
|
||||
*/
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
|
||||
/**
|
||||
* How far the head is tilted up from level, in degrees (negative: down), from
|
||||
* a reading taken at rest. Rolling the head to the side does not change it.
|
||||
*/
|
||||
export function imuPitchDegrees(reading: { x: number; y: number; z: number }): number {
|
||||
return (Math.atan2(reading.x, Math.hypot(reading.y, reading.z)) * 180) / Math.PI;
|
||||
}
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { ApplicationSettings } from '@nativescript/core'
|
||||
import { loadPairedGlassesIdentity } from '../g2/device-addresses'
|
||||
import { g2ManufacturerData } from '../g2/even-advertisement'
|
||||
import { serialKey } from '../g2/glasses-hardware-identity'
|
||||
import { identifyIosPeripheral, type IosAdvertisement, type IosDevice } from '../g2/ios-peripheral-identity'
|
||||
import { hexToBytes } from '../util/hex-util'
|
||||
import { kotlinListener } from './kotlin-listener.ios'
|
||||
@@ -10,6 +13,13 @@ export const BLE_STATE_UNSUPPORTED = 2, BLE_STATE_UNAUTHORIZED = 3, BLE_STATE_PO
|
||||
/** CBManagerAuthorization.allowedAlways. */
|
||||
const BLE_AUTHORIZATION_ALLOWED = 3
|
||||
|
||||
/**
|
||||
* Per address, the peripheral identifier and side last heard there and (since 2026-11-16)
|
||||
* the advertisement itself, from which an arm that has stopped advertising is rebuilt.
|
||||
*/
|
||||
const PERIPHERAL_KEY_PREFIX = 'ios.ble.peripheral.'
|
||||
type RememberedPeripheral = { identifier: string; role: string; name?: string; manufacturerData?: string }
|
||||
|
||||
export type BluetoothEvent =
|
||||
| { kind: 'state'; state: number; authorization: number }
|
||||
| { kind: 'advertisement'; identifier: string; name: string; manufacturerData: string; rssi: number; connectable: boolean }
|
||||
@@ -28,6 +38,8 @@ export class IosBluetooth {
|
||||
private readonly native = FaceclawKitIosBleCentral.alloc().initWithPlatform(FaceclawKitIosProtocolPlatform.shared)
|
||||
private readonly listeners = new Set<(event: BluetoothEvent) => void>()
|
||||
readonly devices = new Map<string, IosDevice>()
|
||||
/** The record last written per settings key, so repeated advertisements don't rewrite it. */
|
||||
private readonly rememberedValues = new Map<string, string>()
|
||||
private scanTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private scanGeneration = 0
|
||||
private readonly scanListener: object
|
||||
@@ -47,12 +59,68 @@ export class IosBluetooth {
|
||||
const device = identifyIosPeripheral(raw)
|
||||
if (device) {
|
||||
this.devices.set(device.address, device)
|
||||
ApplicationSettings.setString(`ios.ble.peripheral.${device.address}`, JSON.stringify({ identifier: device.identifier, role: device.role }))
|
||||
this.remember(device)
|
||||
this.emit({ kind: 'device', device })
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
/** Persists what `device` advertised; skips the write when nothing changed (scans report duplicates). */
|
||||
private remember(device: IosDevice): void {
|
||||
const key = `${PERIPHERAL_KEY_PREFIX}${device.address}`
|
||||
const record: RememberedPeripheral = { identifier: device.identifier, role: device.role, name: device.name, manufacturerData: device.manufacturerData }
|
||||
const value = JSON.stringify(record)
|
||||
if (this.rememberedValues.get(key) === value) return
|
||||
this.rememberedValues.set(key, value)
|
||||
ApplicationSettings.setString(key, value)
|
||||
}
|
||||
/**
|
||||
* Even arms the system is connected to, rebuilt from what they last advertised here. A
|
||||
* connected arm doesn't advertise, and iOS keeps the right arm's ANCS link open after a
|
||||
* session ends (even after pairing other glasses), so a scan alone hears only the left
|
||||
* arm. An arm this phone has never heard advertise can't be rebuilt: its MAC and the
|
||||
* pair's serial come only from the advertisement. The device has no signal strength.
|
||||
*/
|
||||
systemConnectedDevices(): IosDevice[] {
|
||||
if (this.state !== BLE_STATE_POWERED_ON) return []
|
||||
let connected: any
|
||||
try {
|
||||
connected = this.native.systemConnectedG2Arms()
|
||||
} catch (error) {
|
||||
// Scanning still works without this; don't let it take the pairing page down.
|
||||
console.log(`[IosBluetooth] systemConnectedG2Arms failed: ${error}`)
|
||||
return []
|
||||
}
|
||||
if (!connected?.count) return []
|
||||
const records = new Map<string, { address: string; record: RememberedPeripheral }>()
|
||||
for (const key of ApplicationSettings.getAllKeys()) {
|
||||
if (!key.startsWith(PERIPHERAL_KEY_PREFIX)) continue
|
||||
try {
|
||||
const record = JSON.parse(ApplicationSettings.getString(key, '{}')) as RememberedPeripheral
|
||||
if (typeof record.identifier === 'string') records.set(record.identifier.toUpperCase(), { address: key.slice(PERIPHERAL_KEY_PREFIX.length), record })
|
||||
} catch { /* Malformed: a fresh advertisement rewrites it. */ }
|
||||
}
|
||||
const identity = loadPairedGlassesIdentity()
|
||||
const devices: IosDevice[] = []
|
||||
for (let i = 0; i < connected.count; i++) {
|
||||
const peripheral = connected.objectAtIndex(i)
|
||||
const identifier = String(peripheral.identifier)
|
||||
const remembered = records.get(identifier)
|
||||
if (!remembered) continue
|
||||
const { address, record } = remembered
|
||||
let name = record.name ?? ''
|
||||
let manufacturerData = record.manufacturerData ?? ''
|
||||
if (!manufacturerData && identity && (address === identity.leftAddress || address === identity.rightAddress)) {
|
||||
// Remembered before advertisements were: the saved pairing supplies the serial.
|
||||
const side = address === identity.leftAddress ? 'left' : 'right'
|
||||
name ||= side === 'left' ? identity.leftName : identity.rightName
|
||||
manufacturerData = g2ManufacturerData(serialKey(identity.serial) ?? '', address) ?? ''
|
||||
}
|
||||
const device = identifyIosPeripheral({ identifier, name: name || String(peripheral.name ?? ''), manufacturerData, rssi: 0, connectable: true })
|
||||
if (device?.address === address && device.role === record.role) devices.push({ ...device, rssi: null, source: 'paired' })
|
||||
}
|
||||
return devices
|
||||
}
|
||||
onEvent(listener: (event: BluetoothEvent) => void): () => void {
|
||||
this.listeners.add(listener); return () => { this.listeners.delete(listener) }
|
||||
}
|
||||
@@ -111,7 +179,7 @@ export class IosBluetooth {
|
||||
const live = this.devices.get(address)
|
||||
if (live && live.role !== role) throw new Error(`${address} advertises as ${live.role}, not ${role}. Check Configure devices.`)
|
||||
try {
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
|
||||
if (saved.role === role && typeof saved.identifier === 'string') result[role] = saved.identifier
|
||||
} catch { /* A fresh scan can recover malformed settings. */ }
|
||||
}
|
||||
@@ -138,13 +206,15 @@ export class IosBluetooth {
|
||||
/** The peripheral last heard at this address, if it was on this side. */
|
||||
peripheralIdentifier(address: string, role: string): string | null {
|
||||
try {
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
|
||||
return saved.role === role && typeof saved.identifier === 'string' ? saved.identifier : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
forget(address: string): void { ApplicationSettings.remove(`ios.ble.peripheral.${address}`); this.devices.delete(address) }
|
||||
forget(address: string): void {
|
||||
ApplicationSettings.remove(`${PERIPHERAL_KEY_PREFIX}${address}`); this.rememberedValues.delete(`${PERIPHERAL_KEY_PREFIX}${address}`); this.devices.delete(address)
|
||||
}
|
||||
}
|
||||
let instance: IosBluetooth | null = null
|
||||
export function iosBluetooth(): IosBluetooth { return instance ??= new IosBluetooth() }
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/** The iOS twin of keyboard-link.ts, over the running shared-Kotlin session. */
|
||||
|
||||
export function keyboardLinkSession(): unknown {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
|
||||
return (session as FaceclawCommunicatorBridge).sendFaceclawKeyboardControl(record, label);
|
||||
}
|
||||
|
||||
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
|
||||
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
|
||||
const unsubscribe = (session as FaceclawCommunicatorBridge).addKeyboardEventListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { toJavaBytes } from "./cloud-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Platform plumbing for the CFW keyboard bridge (app/g2/keyboard/): the running
|
||||
* session to send field-130 controls on and to hear field-131 records from.
|
||||
* The session is opaque and is a different object after a reconnect.
|
||||
*/
|
||||
|
||||
export function keyboardLinkSession(): unknown {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
|
||||
return (session as any).sendFaceclawKeyboardControl(toJavaBytes(record), label) !== false;
|
||||
}
|
||||
|
||||
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
|
||||
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
|
||||
const communicator = session as any;
|
||||
const proxy = new com.faceclaw.app.FaceclawKeyboardEventListener({
|
||||
onKeyboardEvent: (body: any) => listener(toUint8Array(body)),
|
||||
});
|
||||
communicator.addKeyboardEventListener(proxy);
|
||||
return () => {
|
||||
try {
|
||||
communicator.removeKeyboardEventListener(proxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
|
||||
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
|
||||
declare const FaceclawKitIosMicRecordings: any
|
||||
|
||||
/**
|
||||
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
|
||||
* Documents/Downloads while it is captured, through the shared Kotlin writer
|
||||
* that Android's captures are recorded with. The file is written about a
|
||||
* second at a time, so a crash leaves everything up to the last second.
|
||||
*/
|
||||
export class MicRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/**
|
||||
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
|
||||
* `info` describes the capture and is stored in the file.
|
||||
*/
|
||||
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.mka`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
|
||||
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
|
||||
return recorder ? new MicRecorder(recorder, path) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.recorder || !pcm.length) return
|
||||
// The handle is the buffer's start: a view into a larger buffer is copied out first.
|
||||
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
|
||||
try {
|
||||
// Copied by the recorder before this returns.
|
||||
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
this.close()
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.recorder?.finish() } catch { /* already closed */ }
|
||||
this.recorder = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
|
||||
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
|
||||
export const RECORDING_EXTENSION = ".mka";
|
||||
|
||||
export function isMicRecordingFile(name: string): boolean {
|
||||
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
|
||||
}
|
||||
|
||||
/**
|
||||
* The path for a new recording in the public Downloads directory:
|
||||
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
|
||||
* Downloads isn't available. The file itself is created by whoever records.
|
||||
*/
|
||||
export function newRecordingPath(name: string): string | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
|
||||
return String(file.getAbsolutePath());
|
||||
} catch (error) {
|
||||
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Platform-independent half of the SSH transport bindings (ssh.ts /
|
||||
* ssh.ios.ts): the types the app uses and the decoding of the Go
|
||||
* transport's events (native/ssh/faceclawssh.go, Listener).
|
||||
*/
|
||||
|
||||
/** Where and as whom to connect. */
|
||||
export type SshEndpoint = {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
/** Pinned host keys (SSH wire format, base64). Empty: first connection reports hostkey-unknown. */
|
||||
hostKeys: string[];
|
||||
/** Saved password, or "" to try only the device key. */
|
||||
password: string;
|
||||
/** Offer the device's client key. */
|
||||
useKey: boolean;
|
||||
};
|
||||
|
||||
export type SshChannelSpec =
|
||||
| {
|
||||
/** Run a command on the host; text is exchanged a line at a time. */
|
||||
type: "exec";
|
||||
command: string;
|
||||
}
|
||||
| {
|
||||
/** WebSocket to a URL resolved on the host's side (e.g. ws://127.0.0.1:8737/). */
|
||||
type: "websocket";
|
||||
url: string;
|
||||
};
|
||||
|
||||
export type SshHostKey = {
|
||||
/** SSH key type, e.g. "ssh-ed25519". */
|
||||
type: string;
|
||||
/** OpenSSH-style "SHA256:..." fingerprint. */
|
||||
fingerprint: string;
|
||||
/** SSH wire format, base64: the value to pin in SshEndpoint.hostKeys. */
|
||||
key: string;
|
||||
};
|
||||
|
||||
export type SshFailureKind =
|
||||
| "hostkey-unknown"
|
||||
| "hostkey-mismatch"
|
||||
| "auth"
|
||||
| "network"
|
||||
| "channel"
|
||||
| "config"
|
||||
| "internal";
|
||||
|
||||
export type SshFailure = {
|
||||
kind: SshFailureKind;
|
||||
message: string;
|
||||
/** For hostkey-*: the key the host presented. */
|
||||
hostKey?: SshHostKey;
|
||||
/** For auth: the host takes passwords / we sent one / we offered our key. */
|
||||
passwordAllowed?: boolean;
|
||||
passwordTried?: boolean;
|
||||
keyTried?: boolean;
|
||||
};
|
||||
|
||||
export type SshClosed = {
|
||||
/** Exec channels: the command's exit status (and signal, if killed). */
|
||||
exitStatus?: number;
|
||||
signal?: string;
|
||||
/** WebSocket channels: the close frame's code and reason. */
|
||||
code?: number;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
export type SshChannelListener = {
|
||||
onOpen(): void;
|
||||
onText(text: string): void;
|
||||
onStderr?(line: string): void;
|
||||
/** Terminal: the channel ended normally. */
|
||||
onClosed(info: SshClosed): void;
|
||||
/** Terminal: the channel couldn't open or was lost. */
|
||||
onFailure(failure: SshFailure): void;
|
||||
};
|
||||
|
||||
export type SshChannel = {
|
||||
/** Queue a websocket text frame / an exec stdin line. False once closed. */
|
||||
send(text: string): boolean;
|
||||
/** Flush queued sends, then end the channel. No events follow. */
|
||||
close(): void;
|
||||
};
|
||||
|
||||
export type SshClientKey = {
|
||||
/** "ecdsa-sha2-nistp256 AAAA... comment", for ~/.ssh/authorized_keys. */
|
||||
authorizedKey: string;
|
||||
fingerprint: string;
|
||||
};
|
||||
|
||||
/** Build the Go transport's openSpec JSON. */
|
||||
export function encodeSshSpec(endpoint: SshEndpoint, channel: SshChannelSpec): string {
|
||||
return JSON.stringify({
|
||||
host: endpoint.host,
|
||||
port: endpoint.port,
|
||||
user: endpoint.user,
|
||||
password: endpoint.password,
|
||||
hostKeys: endpoint.hostKeys,
|
||||
useKey: endpoint.useKey,
|
||||
channel,
|
||||
});
|
||||
}
|
||||
|
||||
/** Route one transport event to the listener. Unknown kinds are ignored. */
|
||||
export function dispatchSshEvent(listener: SshChannelListener, kind: string, payload: string): void {
|
||||
switch (kind) {
|
||||
case "open":
|
||||
listener.onOpen();
|
||||
return;
|
||||
case "text":
|
||||
listener.onText(payload);
|
||||
return;
|
||||
case "stderr":
|
||||
listener.onStderr?.(payload);
|
||||
return;
|
||||
case "closed":
|
||||
listener.onClosed(parseJson(payload) ?? {});
|
||||
return;
|
||||
case "failure": {
|
||||
const failure = parseJson(payload);
|
||||
listener.onFailure(
|
||||
failure && typeof failure.kind === "string"
|
||||
? failure
|
||||
: { kind: "internal", message: `bad failure payload: ${payload}` },
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseJson(text: string): any {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Human summary of a failure for status lines. */
|
||||
export function describeSshFailure(failure: SshFailure): string {
|
||||
switch (failure.kind) {
|
||||
case "hostkey-unknown":
|
||||
return "Host key not yet trusted.";
|
||||
case "hostkey-mismatch":
|
||||
return "HOST KEY CHANGED; refusing to connect.";
|
||||
case "auth":
|
||||
return failure.message || "Authentication failed.";
|
||||
default:
|
||||
return failure.message || failure.kind;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
dispatchSshEvent,
|
||||
encodeSshSpec,
|
||||
type SshChannel,
|
||||
type SshChannelListener,
|
||||
type SshChannelSpec,
|
||||
type SshClientKey,
|
||||
type SshEndpoint,
|
||||
} from "./ssh-common";
|
||||
|
||||
export * from "./ssh-common";
|
||||
|
||||
declare const FaceclawSshChannel: any;
|
||||
declare const FaceclawSshKeys: any;
|
||||
|
||||
// Like socket.ios.ts: Go delivers events on its own threads, so the native
|
||||
// side queues them and this isolate drains the queue on a timer.
|
||||
const POLL_INTERVAL_MS = 25;
|
||||
|
||||
/** iOS SSH transport: FaceclawSshChannel (ObjC) over the gomobile-bound Go package in native/ssh. */
|
||||
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
|
||||
const native = FaceclawSshChannel.alloc().initWithSpecUseKey(encodeSshSpec(endpoint, channel), endpoint.useKey);
|
||||
let ended = false;
|
||||
const stop = () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
clearInterval(timer);
|
||||
native.close();
|
||||
};
|
||||
const timer = setInterval(() => {
|
||||
const events = JSON.parse(String(native.takeEvents())) as Array<{ kind: string; payload: string }>;
|
||||
for (const event of events) {
|
||||
if (ended) break;
|
||||
if (event.kind === "closed" || event.kind === "failure") {
|
||||
// Terminal events: stop polling, then deliver.
|
||||
ended = true;
|
||||
clearInterval(timer);
|
||||
}
|
||||
dispatchSshEvent(listener, event.kind, event.payload);
|
||||
}
|
||||
}, POLL_INTERVAL_MS);
|
||||
return {
|
||||
send: (text) => !ended && native.send(text) === true,
|
||||
close: stop,
|
||||
};
|
||||
}
|
||||
|
||||
export function sshClientKey(comment: string): SshClientKey {
|
||||
const authorizedKey = FaceclawSshKeys.authorizedKeyWithComment(comment);
|
||||
const fingerprint = FaceclawSshKeys.fingerprint();
|
||||
if (!authorizedKey || !fingerprint) throw new Error("Could not create the SSH client key.");
|
||||
return { authorizedKey: String(authorizedKey), fingerprint: String(fingerprint) };
|
||||
}
|
||||
|
||||
export function getSshSecret(id: string): string {
|
||||
return String(FaceclawSshKeys.secretForId(id) ?? "");
|
||||
}
|
||||
|
||||
export function setSshSecret(id: string, value: string): void {
|
||||
FaceclawSshKeys.setSecretForId(value, id);
|
||||
}
|
||||
|
||||
export function removeSshSecret(id: string): void {
|
||||
FaceclawSshKeys.removeSecretForId(id);
|
||||
}
|
||||
|
||||
export function copyToPhoneClipboard(_label: string, text: string): void {
|
||||
FaceclawSshKeys.copyToClipboard(text);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import {
|
||||
dispatchSshEvent,
|
||||
encodeSshSpec,
|
||||
type SshChannel,
|
||||
type SshChannelListener,
|
||||
type SshChannelSpec,
|
||||
type SshClientKey,
|
||||
type SshEndpoint,
|
||||
} from "./ssh-common";
|
||||
|
||||
export * from "./ssh-common";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Android SSH transport: FaceclawSshChannel (Kotlin) over the gomobile-bound
|
||||
* Go package in native/ssh. Callable from the main isolate or a worker;
|
||||
* events arrive on the calling isolate's thread.
|
||||
*/
|
||||
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
|
||||
let closed = false;
|
||||
const proxy = new com.faceclaw.app.FaceclawSshListener({
|
||||
onEvent: (kind: string, payload: string) => {
|
||||
if (closed) return;
|
||||
dispatchSshEvent(listener, String(kind), payload == null ? "" : String(payload));
|
||||
},
|
||||
});
|
||||
const native = new com.faceclaw.app.FaceclawSshChannel(encodeSshSpec(endpoint, channel), endpoint.useKey, proxy);
|
||||
return {
|
||||
send: (text) => !closed && native.send(text) === true,
|
||||
close: () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
native.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function context(): any {
|
||||
return Utils.android.getApplicationContext();
|
||||
}
|
||||
|
||||
/** The device's SSH client key (generated on first use). */
|
||||
export function sshClientKey(comment: string): SshClientKey {
|
||||
return {
|
||||
authorizedKey: String(com.faceclaw.app.FaceclawSshKeys.authorizedKey(comment)),
|
||||
fingerprint: String(com.faceclaw.app.FaceclawSshKeys.fingerprint()),
|
||||
};
|
||||
}
|
||||
|
||||
export function getSshSecret(id: string): string {
|
||||
return String(com.faceclaw.app.FaceclawSshKeys.getSecret(context(), id) ?? "");
|
||||
}
|
||||
|
||||
export function setSshSecret(id: string, value: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.setSecret(context(), id, value);
|
||||
}
|
||||
|
||||
export function removeSshSecret(id: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.removeSecret(context(), id);
|
||||
}
|
||||
|
||||
export function copyToPhoneClipboard(label: string, text: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.copyToClipboard(context(), label, text);
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Session-list and connection-state types shared by the Terminal app's
|
||||
* backends (g2mirror-client.ts, apps/terminal/tmux-client.ts). Kept free of
|
||||
* platform imports so the tmux client can be unit-tested under node.
|
||||
*/
|
||||
|
||||
export type G2MirrorSession = {
|
||||
socket: string;
|
||||
pid: number;
|
||||
cwdHint: string;
|
||||
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
|
||||
lastBellAt: number | null;
|
||||
/**
|
||||
* Unix epoch ms (host clock) of the app's last output, or null if none
|
||||
* observed. Fresh to within the wrapper's 2s activity reporting interval
|
||||
* while we're connected; compare against other host timestamps, not the
|
||||
* phone clock (see the worker's local-receive-time activity tracking).
|
||||
*/
|
||||
lastOutputAt: number | null;
|
||||
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
|
||||
title: string | null;
|
||||
};
|
||||
|
||||
export type G2MirrorPhase =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "connected" // handshake accepted; can list/attach
|
||||
| "attached" // relaying to one session
|
||||
| "failed";
|
||||
|
||||
export type G2MirrorState = {
|
||||
phase: G2MirrorPhase;
|
||||
status: string;
|
||||
sessions: G2MirrorSession[];
|
||||
attachedCommand: string;
|
||||
/**
|
||||
* server_name from the init success reply: a human-readable name for the
|
||||
* machine the server runs on. "" until the handshake succeeds (or when an
|
||||
* older server omits the field).
|
||||
*/
|
||||
serverName: string;
|
||||
/**
|
||||
* False when reconnecting can't help until the user changes something on
|
||||
* the host (e.g. tmux missing or too old). Absent means retryable.
|
||||
*/
|
||||
retryable?: boolean;
|
||||
};
|
||||
|
||||
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
|
||||
export type G2MirrorHistoryReply = {
|
||||
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
|
||||
start: number;
|
||||
/** Oldest index still retained by the wrapper (older requests return nothing). */
|
||||
oldest: number;
|
||||
/** Current splice point (next line to be archived). */
|
||||
next: number;
|
||||
/** Plain text, ANSI-stripped, oldest-to-newest. */
|
||||
lines: string[];
|
||||
};
|
||||
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { MicRecorder } from './mic-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
|
||||
private continuous = false
|
||||
// Shown instead of the native "ended" status once a drain the bridge started completes.
|
||||
private drainStatus: string | null = null
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
private recorder: MicRecorder | null = null
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false
|
||||
private readonly recordingListeners = new Set<() => void>()
|
||||
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or ''. Only cloud captures supply PCM.
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
|
||||
private record(pcm: Uint8Array): void {
|
||||
if (!this.recording) return
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName)
|
||||
this.recorder = MicRecorder.open(this.recordingName, {
|
||||
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
|
||||
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
|
||||
})
|
||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||
}
|
||||
|
||||
+319
-27
@@ -1,10 +1,10 @@
|
||||
import type { OpenAiSttModel } from "./openai-stt";
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { newRecordingPath } from "./mic-recording";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { WavRecorder } from "./wav-recorder";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
@@ -86,6 +86,49 @@ export type PushToTalkOptions = {
|
||||
* frame). Null/absent leaves the mic omnidirectional.
|
||||
*/
|
||||
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
|
||||
/**
|
||||
* Capture from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the glasses' mixed stream. Each temple
|
||||
* streams its microphones on its own link and clock; the phone mixes them
|
||||
* into the mono feed that is transcribed. Null/absent keeps the mixed
|
||||
* stream; ignored with usePhoneMic.
|
||||
*/
|
||||
micArray?: MicArrayCapture | null;
|
||||
/**
|
||||
* Play this recording (a .mka file from Save Recording) in real time in
|
||||
* place of any microphone. The capture needs no glasses, and ends by
|
||||
* itself when the recording does.
|
||||
*/
|
||||
replay?: { path: string; startMs: number } | null;
|
||||
/** What the caller knows of the capture, for the description stored in a recording of it. */
|
||||
captureInfo?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type MicArrayCapture = {
|
||||
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
|
||||
configure: Uint8Array;
|
||||
};
|
||||
|
||||
/** Progress of a capture that replays a recording; positions are ms into it. */
|
||||
export type VoiceReplayEvent = {
|
||||
/**
|
||||
* `start` once the file is open, `position` about once per second played,
|
||||
* `seek` when a jump took effect, `end` when the recording ran out, and
|
||||
* `error` when it could not be opened (the status says why).
|
||||
*/
|
||||
kind: "start" | "position" | "seek" | "end" | "error";
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
|
||||
export type VoiceEventSource = {
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void;
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
|
||||
onSpeechPause(listener: () => void): () => void;
|
||||
onMicLevel(listener: (level: number) => void): () => void;
|
||||
readonly recording: boolean;
|
||||
readonly recordingFailed: boolean;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
@@ -132,9 +175,13 @@ export class FaceclawVoiceControlBridge {
|
||||
private captureEndpointing = false;
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// What the running STT stream transcribes with (transcriptionKey); a holder
|
||||
// asking for something else restarts the stream rather than sharing it.
|
||||
// What the running STT stream captures from and transcribes with
|
||||
// (streamKey); a holder asking for something else restarts the stream
|
||||
// rather than sharing it.
|
||||
private streamProvider: string | null = null;
|
||||
// The options of the running stream, for the description of a recording of it.
|
||||
private streamOptions: PushToTalkOptions | null = null;
|
||||
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
|
||||
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
|
||||
// controller runs in the decode-only "cloud" mode and every decoded PCM
|
||||
// frame is broadcast to these listeners. STT capture preempts it — a live
|
||||
@@ -152,11 +199,14 @@ export class FaceclawVoiceControlBridge {
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set: every source's chunks as they arrive, before any processing, written
|
||||
// by the controller. The file outlives a capture, so a restarted one keeps
|
||||
// adding to it, and it is created by the first audio, so a failed start
|
||||
// leaves no file.
|
||||
private recordingName = "";
|
||||
private recorder: WavRecorder | null = null;
|
||||
// The file is open in the controller.
|
||||
private recordingOpen = false;
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false;
|
||||
private readonly recordingListeners = new Set<() => void>();
|
||||
@@ -253,16 +303,55 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* End the continuous capture as an utterance: stop it and have the
|
||||
* transcript of its last audio come as a final. For a capture whose audio
|
||||
* ran out by itself (a replay).
|
||||
*/
|
||||
finishContinuousCapture(): void {
|
||||
this.releaseCapture("continuous", true);
|
||||
}
|
||||
|
||||
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
|
||||
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
|
||||
this.replayListeners.add(listener);
|
||||
return () => this.replayListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Continue the running replay from `positionMs` into its recording. */
|
||||
seekReplay(positionMs: number): void {
|
||||
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. */
|
||||
setReplayPaused(paused: boolean): void {
|
||||
this.controller?.setReplayPaused(paused);
|
||||
}
|
||||
|
||||
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
|
||||
isCaptureLive(): boolean {
|
||||
return this.micIsLive();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or "".
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return;
|
||||
this.recorder?.close();
|
||||
this.recorder = null;
|
||||
this.closeRecording();
|
||||
this.setRecording(name, false);
|
||||
}
|
||||
|
||||
private closeRecording(): void {
|
||||
if (!this.recordingOpen) return;
|
||||
this.recordingOpen = false;
|
||||
try {
|
||||
this.controller?.stopRecording();
|
||||
} catch (error) {
|
||||
console.warn(`Voice: closing the recording failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean {
|
||||
return this.recordingName !== "" && !this.recordingFailure;
|
||||
@@ -288,18 +377,48 @@ export class FaceclawVoiceControlBridge {
|
||||
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||
}
|
||||
|
||||
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||
private record(pcm: any): void {
|
||||
if (!this.recording) return;
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName);
|
||||
if (!this.recorder) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||
/**
|
||||
* Audio arrived for a continuous capture: open the recording asked for, if
|
||||
* it isn't open yet. The controller writes it from then on, as the audio
|
||||
* arrives on its own thread.
|
||||
*/
|
||||
private ensureRecording(): void {
|
||||
if (!this.recording || this.recordingOpen) return;
|
||||
const path = newRecordingPath(this.recordingName);
|
||||
let opened = false;
|
||||
try {
|
||||
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: could not create the recording: ${error}`);
|
||||
}
|
||||
this.recorder.append(pcm);
|
||||
if (!opened) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
this.recordingOpen = true;
|
||||
console.log(`Voice: recording audio to ${path}`);
|
||||
}
|
||||
|
||||
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
|
||||
private syncRecordingEnabled(): void {
|
||||
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
/** The running capture, as the JSON stored in a recording of it. */
|
||||
private describeCapture(): Record<string, unknown> {
|
||||
const options = this.streamOptions;
|
||||
return {
|
||||
recording: this.recordingName,
|
||||
startedAt: new Date().toISOString(),
|
||||
source: options?.usePhoneMic ? "phone" : "glasses",
|
||||
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
|
||||
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
|
||||
processing: {
|
||||
noiseSuppression: Boolean(options?.noiseSuppression),
|
||||
beamFilter: options?.beamFilter ?? null,
|
||||
},
|
||||
...options?.captureInfo,
|
||||
};
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
@@ -366,6 +485,8 @@ export class FaceclawVoiceControlBridge {
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
this.controller?.setMicArray(null);
|
||||
this.controller?.setReplay(null, 0);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
this.controller?.setNoiseSuppression(false);
|
||||
@@ -401,18 +522,33 @@ export class FaceclawVoiceControlBridge {
|
||||
const micLive = this.micIsLive();
|
||||
this.suspendedHolders.delete(holder);
|
||||
this.captureHolders.add(holder);
|
||||
if (micLive && this.streamProvider === transcriptionKey(options)) {
|
||||
if (micLive && this.streamProvider === streamKey(options)) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
this.syncRecordingEnabled();
|
||||
return;
|
||||
}
|
||||
// Any leftover capture is dead, or transcribes with another provider than
|
||||
// Any leftover capture is dead, or captures or transcribes otherwise than
|
||||
// this holder needs; drop it before starting the new one.
|
||||
if (this.started) this.teardownCapture();
|
||||
this.streamProvider = transcriptionKey(options);
|
||||
this.streamProvider = streamKey(options);
|
||||
this.streamOptions = options;
|
||||
this.ensureController();
|
||||
this.syncRecordingEnabled();
|
||||
// A recording under way carries on into this capture: note what changed.
|
||||
if (this.recordingOpen) {
|
||||
try {
|
||||
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: recording event failed: ${error}`);
|
||||
}
|
||||
}
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setMicArray(
|
||||
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
|
||||
);
|
||||
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
@@ -437,7 +573,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.speechPause.reset();
|
||||
// Replaces whatever the last capture left behind ("Listening...", an
|
||||
// error, the permission prompt) until the controller reports progress.
|
||||
this.setStatus("Starting microphone...");
|
||||
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
@@ -485,6 +621,7 @@ export class FaceclawVoiceControlBridge {
|
||||
if (commit) this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
this.syncRecordingEnabled();
|
||||
if (this.captureHolders.size > 0) {
|
||||
// Another holder still wants the mic; keep it running.
|
||||
return;
|
||||
@@ -509,6 +646,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.captureHolders.clear();
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
this.syncRecordingEnabled();
|
||||
this.teardownCapture();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
@@ -526,6 +664,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.suspendedHolders.add(holder);
|
||||
}
|
||||
this.captureHolders.clear();
|
||||
this.syncRecordingEnabled();
|
||||
this.suspendedRaw = this.rawActive;
|
||||
this.teardownCapture();
|
||||
if (this.suspendedHolders.size > 0) {
|
||||
@@ -583,6 +722,7 @@ export class FaceclawVoiceControlBridge {
|
||||
private teardownCapture(): void {
|
||||
this.rawActive = false;
|
||||
this.streamProvider = null;
|
||||
this.streamOptions = null;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
@@ -616,7 +756,7 @@ export class FaceclawVoiceControlBridge {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||
if (this.captureHolders.has("continuous")) this.ensureRecording();
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
@@ -647,6 +787,10 @@ export class FaceclawVoiceControlBridge {
|
||||
);
|
||||
}
|
||||
},
|
||||
onReplay: (kind: string, positionMs: number, durationMs: number) => {
|
||||
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
|
||||
for (const listener of Array.from(this.replayListeners)) listener(event);
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
@@ -683,6 +827,21 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture reads and transcribes with. A running capture is shared only
|
||||
* by holders asking for the same: the provider and language, the microphones
|
||||
* (the mixed stream, or the temples' own under a given configuration), or the
|
||||
* recording it replays.
|
||||
*/
|
||||
function streamKey(options: PushToTalkOptions): string {
|
||||
const source = options.replay
|
||||
? `replay:${options.replay.path}`
|
||||
: options.micArray && !options.usePhoneMic
|
||||
? `array:${Array.from(options.micArray.configure).join(",")}`
|
||||
: "mixed";
|
||||
return `${transcriptionKey(options)}|${source}`;
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
const javaFloats = Array.create("float", values.length);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
@@ -692,3 +851,136 @@ function toJavaFloats(values: Float32Array): any {
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
|
||||
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
|
||||
|
||||
/**
|
||||
* A recording played back as a capture of its own: transcribed by the same
|
||||
* pipeline as the microphone, with the same settings, on a bridge and native
|
||||
* controller separate from the microphone's. Nothing here touches the mic,
|
||||
* so voice input and a live Transcribe window carry on beside a replay.
|
||||
*/
|
||||
export class VoiceReplay {
|
||||
private readonly bridge = new FaceclawVoiceControlBridge();
|
||||
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
|
||||
private readonly offReplay: () => void;
|
||||
private position = 0;
|
||||
private duration = 0;
|
||||
private current: VoiceReplayState = "opening";
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* `options` gives the capture options (provider, keys, processing) each
|
||||
* time the replay starts, so a restart picks up changed settings.
|
||||
*/
|
||||
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
|
||||
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
|
||||
}
|
||||
|
||||
/** The replay's status, transcript and level events. */
|
||||
get events(): VoiceEventSource {
|
||||
return this.bridge;
|
||||
}
|
||||
|
||||
/** Milliseconds into the recording. */
|
||||
get positionMs(): number {
|
||||
return this.position;
|
||||
}
|
||||
|
||||
/** The recording's length in ms; 0 until it has been opened. */
|
||||
get durationMs(): number {
|
||||
return this.duration;
|
||||
}
|
||||
|
||||
get state(): VoiceReplayState {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/**
|
||||
* The position, duration or state changed. `jumpedToMs` is given when
|
||||
* playback moved there other than by playing on: a jump, or a start
|
||||
* part-way in. It comes after the transcript of what played before.
|
||||
*/
|
||||
onChange(listener: (jumpedToMs?: number) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Play from `positionMs` into the recording. */
|
||||
start(positionMs = 0): void {
|
||||
if (this.closed) return;
|
||||
this.position = Math.max(0, positionMs);
|
||||
this.set("opening");
|
||||
this.bridge.startContinuousCapture({
|
||||
...this.options(),
|
||||
communicator: null,
|
||||
usePhoneMic: false,
|
||||
micArray: null,
|
||||
replay: { path: this.path, startMs: this.position },
|
||||
});
|
||||
}
|
||||
|
||||
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
|
||||
jumpTo(positionMs: number): void {
|
||||
if (this.closed) return;
|
||||
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
|
||||
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
|
||||
this.bridge.seekReplay(target);
|
||||
this.position = target;
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.start(target);
|
||||
}
|
||||
|
||||
setPaused(paused: boolean): void {
|
||||
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
|
||||
this.bridge.setReplayPaused(paused);
|
||||
this.set(paused ? "paused" : "playing");
|
||||
}
|
||||
|
||||
/** Stop for good and release the capture. */
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.offReplay();
|
||||
this.listeners.clear();
|
||||
this.bridge.stop();
|
||||
}
|
||||
|
||||
private handle(event: VoiceReplayEvent): void {
|
||||
this.duration = event.durationMs;
|
||||
switch (event.kind) {
|
||||
case "start":
|
||||
this.position = event.positionMs;
|
||||
this.current = "playing";
|
||||
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
|
||||
return;
|
||||
case "seek":
|
||||
this.position = event.positionMs;
|
||||
this.notify(event.positionMs);
|
||||
return;
|
||||
case "end":
|
||||
this.position = event.durationMs;
|
||||
// Its last audio is still to be transcribed (a cloud provider's final).
|
||||
this.bridge.finishContinuousCapture();
|
||||
this.set("ended");
|
||||
return;
|
||||
case "error":
|
||||
this.set("failed");
|
||||
return;
|
||||
default:
|
||||
this.position = event.positionMs;
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private set(state: VoiceReplayState): void {
|
||||
this.current = state;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
private notify(jumpedToMs?: number): void {
|
||||
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
|
||||
export function wavHeader(dataBytes: number): Uint8Array {
|
||||
const header = new Uint8Array(44);
|
||||
const view = new DataView(header.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataBytes, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, 16000, true);
|
||||
view.setUint32(28, 32000, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, dataBytes, true);
|
||||
return header;
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
import { wavHeader } from './wav-header'
|
||||
|
||||
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
|
||||
const FLUSH_BYTES = 32000
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
|
||||
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private pending: Uint8Array[] = []
|
||||
private pendingBytes = 0
|
||||
private dataBytes = 0
|
||||
|
||||
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.wav`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
|
||||
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
|
||||
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
|
||||
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
|
||||
return new WavRecorder(handle, path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.handle || !pcm.length) return
|
||||
this.pending.push(pcm.slice())
|
||||
this.pendingBytes += pcm.length
|
||||
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.flush()
|
||||
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
|
||||
this.handle = null
|
||||
}
|
||||
|
||||
private flush(): void {
|
||||
const handle = this.handle
|
||||
if (!handle || !this.pendingBytes) return
|
||||
const chunk = new Uint8Array(this.pendingBytes)
|
||||
let offset = 0
|
||||
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
|
||||
this.pending = []
|
||||
this.pendingBytes = 0
|
||||
try {
|
||||
// NativeScript throws a failed call's NSError (storage full, file removed).
|
||||
handle.writeDataError(toData(chunk))
|
||||
handle.seekToOffsetError(0)
|
||||
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
|
||||
handle.seekToEndReturningOffsetError(null)
|
||||
this.dataBytes += chunk.length
|
||||
return
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
}
|
||||
try { handle.closeAndReturnError() } catch { /* ignore */ }
|
||||
this.handle = null
|
||||
}
|
||||
}
|
||||
|
||||
function toData(bytes: Uint8Array): NSData {
|
||||
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
declare const com: any;
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}.wav`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||
const path = String(file.getAbsolutePath());
|
||||
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||
} catch (error) {
|
||||
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||
append(pcm: any): void {
|
||||
this.recorder?.append(pcm);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.recorder?.finish();
|
||||
this.recorder = null;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user