[verified] merge upstream 25cb255b into all-features

This commit is contained in:
MasterControl committed 2026-11-19 22:21:26 +00:00
commit ffcdae660d
169 files changed
+16201 -597

No files matched your search

+20 -2
View File
@@ -3,11 +3,18 @@ import type { DiscoveryEvents } from './device-discovery'
import type { RawAdvertisement } from '../g2/even-advertisement'
export { buildAddressSet } from './device-discovery-common'
/**
* How often a scan re-reads which arms the system is connected to. Each read refreshes
* their rows, which age out like advertisements once the link drops and they go quiet.
*/
const CONNECTED_ARMS_POLL_MS = 2000
export class DeviceDiscoveryBridge {
private ble = iosBluetooth()
private off: (() => void) | null = null
private generation = 0
private cancelCollection: (() => void) | null = null
private connectedArmsTimer: ReturnType<typeof setInterval> | null = null
get bluetoothEnabled(): boolean { return this.ble.state === 5 }
get scanning(): boolean { return !!this.off && this.ble.scanning }
startScan(events: DiscoveryEvents): boolean {
@@ -21,16 +28,27 @@ export class DeviceDiscoveryBridge {
void this.ble.startScan(24 * 60 * 60 * 1000).catch(error => {
if (generation === this.generation) events.onScanFailed?.(-1, String(error.message ?? error))
})
// A connected arm doesn't advertise (iOS holds the right arm's ANCS link open after a
// session ends), so the scan alone would show those glasses one-armed.
const reportConnectedArms = () => { for (const device of this.ble.systemConnectedDevices()) events.onAdvertisement(device) }
reportConnectedArms()
this.connectedArmsTimer = setInterval(reportConnectedArms, CONNECTED_ARMS_POLL_MS)
return true
}
stopScan(): void {
++this.generation; this.cancelCollection?.()
if (this.connectedArmsTimer) { clearInterval(this.connectedArmsTimer); this.connectedArmsTimer = null }
if (this.off) { this.off(); this.off = null; this.ble.stopScan() }
}
// CoreBluetooth does not expose the phone's bond list. Fresh advertisements
// supply both identity and the UUID mapping that iOS needs for connections.
// supply both identity and the UUID mapping that iOS needs for connections;
// startScan() reports the arms the system is connected to.
emitBondedDevices(): void {}
async getBondedCandidates(): Promise<RawAdvertisement[]> { return [...this.ble.devices.values()] }
async getBondedCandidates(): Promise<RawAdvertisement[]> {
const devices = new Map(this.ble.devices)
for (const device of this.ble.systemConnectedDevices()) if (!devices.has(device.address)) devices.set(device.address, device)
return [...devices.values()]
}
async scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
const generation = this.generation
await this.ble.ensureReady()
+21
View File
@@ -19,6 +19,7 @@ declare const FaceclawKitFaceclawCompassListener: any;
declare const FaceclawKitFaceclawImuListener: any;
declare const FaceclawKitIosAmbientLightListener: any;
declare const FaceclawKitIosMicStatusListener: any;
declare const FaceclawKitIosKeyboardEventListener: any;
export type CommunicatorPhase =
@@ -198,6 +199,13 @@ class MicStatusListener extends NSObject {
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
}
@NativeClass()
class KeyboardEventListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosKeyboardEventListener];
handler!: (body: Uint8Array) => void;
onNsDataKeyboardEventBody(body: NSData): void { this.handler(fromData(body)); }
}
/**
* The iOS twin of the Android communicator bridge: the same public API over
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
@@ -803,6 +811,19 @@ export class FaceclawCommunicatorBridge {
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
}
/** Raw field-131 keyboard-bridge records, in arrival order, on the main queue. */
addKeyboardEventListener(listener: (body: Uint8Array) => void): () => void {
const proxy = KeyboardEventListener.new() as KeyboardEventListener;
proxy.handler = listener;
const handle = this.communicator.addKeyboardEventListenerNsDataListener(proxy);
return () => { this.communicator.removeKeyboardEventListenerListener(handle); };
}
/** Queue a CFW keyboard-bridge control for the right temple; false when nothing was queued. */
sendFaceclawKeyboardControl(record: Uint8Array, label: string): boolean {
return Boolean(this.communicator.sendFaceclawKeyboardControlRecordLabel(toData(new Uint8Array(record)), label));
}
/**
* Every audio-characteristic packet, whatever enabled the mics, on the
* main queue. Passive: sends nothing and doesn't affect capture.
+12 -51
View File
@@ -1,5 +1,6 @@
import { openSocket } from "./socket";
import { openSocket, type SocketConnection, type SocketListener } from "./socket";
import { toUint8Array } from "../util/array-util";
import type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState } from "./terminal-session-types";
declare const android: any;
declare const java: any;
@@ -19,54 +20,7 @@ const LAUNCH_REPLY_TIMEOUT_MS = 10_000;
// otherwise treat the "\r" as a pasted newline instead of the Enter key.
const SUBMIT_DELAY_MS = 150;
export type G2MirrorSession = {
socket: string;
pid: number;
cwdHint: string;
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
lastBellAt: number | null;
/**
* Unix epoch ms (host clock) of the app's last output, or null if none
* observed. Fresh to within the wrapper's 2s activity reporting interval
* while we're connected; compare against other host timestamps, not the
* phone clock (see the worker's local-receive-time activity tracking).
*/
lastOutputAt: number | null;
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
title: string | null;
};
export type G2MirrorPhase =
| "idle"
| "connecting"
| "connected" // handshake accepted; can list/attach
| "attached" // relaying to one session
| "failed";
export type G2MirrorState = {
phase: G2MirrorPhase;
status: string;
sessions: G2MirrorSession[];
attachedCommand: string;
/**
* server_name from the init success reply: a human-readable name for the
* machine the server runs on. "" until the handshake succeeds (or when an
* older server omits the field).
*/
serverName: string;
};
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
export type G2MirrorHistoryReply = {
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
start: number;
/** Oldest index still retained by the wrapper (older requests return nothing). */
oldest: number;
/** Current splice point (next line to be archived). */
next: number;
/** Plain text, ANSI-stripped, oldest-to-newest. */
lines: string[];
};
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
export type G2MirrorClientOptions = {
/** TLS (g2mirrors:// → wss) vs plain (g2mirror:// → ws). */
@@ -77,6 +31,12 @@ export type G2MirrorClientOptions = {
deviceName: string;
cols: number;
rows: number;
/**
* Opens the websocket some other way than dialing host:port directly
* (e.g. through an SSH tunnel); `label` then names the endpoint in status
* lines instead of host:port.
*/
transport?: { open: (listener: SocketListener) => SocketConnection; label: string };
};
type TerminalDataKind = "snapshot" | "output";
@@ -180,7 +140,8 @@ export class G2MirrorClient {
this.stopped = false;
const generation = ++this.connectionGeneration;
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
const transport = this.options.transport;
this.setState("connecting", `Connecting to ${transport ? transport.label : `${this.options.host}:${this.options.port}`}...`);
this.listenerProxy = {
onOpen: () => {
if (this.stopped || generation !== this.connectionGeneration) return;
@@ -208,7 +169,7 @@ export class G2MirrorClient {
},
};
try {
this.ws = openSocket(url, this.listenerProxy);
this.ws = transport ? transport.open(this.listenerProxy!) : openSocket(url, this.listenerProxy);
} catch (error) {
this.ws = null;
this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
+65
View File
@@ -0,0 +1,65 @@
declare const com: any;
/**
* Head-up (look-up wake) trigger on the active glasses session
* (GlassesSessionCore.setHeadUpTrigger / getHeadUpStatus). Android only for
* now (headUpControlsAvailable): on iOS, or with no session, the setter does
* nothing and the getter returns null.
*/
/** Whether this platform can change the glasses' head-up setting at all. */
export function headUpControlsAvailable(): boolean {
return Boolean(global.isAndroid);
}
/** GlassesSessionCore.getHeadUpStatus. switch -1 = not read on this connection. */
export type HeadUpStatus = {
switch: number;
angle: number;
calibration: number;
/** The trigger Faceclaw is maintaining, or -1 when it leaves the glasses' own setting. */
desired: number;
readAgeMs: number;
/** The glasses' own switch and angle from before Faceclaw changed them (switch -1 = none held). */
originalSwitch: number;
originalAngle: number;
/** Those originals are being written back. */
restorePending: boolean;
};
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/**
* Ask the session to keep the head-up trigger at `degrees` (1..90); 0 stops
* managing it and writes back `original` (or what the session saved itself)
* if Faceclaw changed the glasses' values.
*/
export function setHeadUpTrigger(degrees: number, original: { switch: number; angle: number } | null = null): boolean {
const active = activeCommunicator();
if (!active) return false;
try {
active.setHeadUpTrigger(Math.round(degrees), original ? original.switch : -1, original ? Math.round(original.angle) : 0);
return true;
} catch (error) {
console.warn(`setHeadUpTrigger failed: ${error}`);
return false;
}
}
export function getHeadUpStatus(): HeadUpStatus | null {
const active = activeCommunicator();
if (!active) return null;
try {
return JSON.parse(String(active.getHeadUpStatus())) as HeadUpStatus;
} catch (error) {
console.warn(`getHeadUpStatus failed: ${error}`);
return null;
}
}
+16
View File
@@ -1,7 +1,23 @@
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
/**
* An accelerometer sample in g, in Faceclaw's frame for every glasses model
* (the G2's own; the session rotates other models' samples into it, see
* ImuOrientation.kt): +x forward, +y to the wearer's left, +z up through the
* top of the head. At rest it is the "up" direction: head level reads
* (0, 0, 1), looking 45° up (0.71, 0, 0.71), the head tilted 45° to the right
* (0, 0.71, 0.71).
*/
export type ImuReading = { x: number; y: number; z: number; source: number };
/**
* How far the head is tilted up from level, in degrees (negative: down), from
* a reading taken at rest. Rolling the head to the side does not change it.
*/
export function imuPitchDegrees(reading: { x: number; y: number; z: number }): number {
return (Math.atan2(reading.x, Math.hypot(reading.y, reading.z)) * 180) / Math.PI;
}
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
+74 -4
View File
@@ -1,4 +1,7 @@
import { ApplicationSettings } from '@nativescript/core'
import { loadPairedGlassesIdentity } from '../g2/device-addresses'
import { g2ManufacturerData } from '../g2/even-advertisement'
import { serialKey } from '../g2/glasses-hardware-identity'
import { identifyIosPeripheral, type IosAdvertisement, type IosDevice } from '../g2/ios-peripheral-identity'
import { hexToBytes } from '../util/hex-util'
import { kotlinListener } from './kotlin-listener.ios'
@@ -10,6 +13,13 @@ export const BLE_STATE_UNSUPPORTED = 2, BLE_STATE_UNAUTHORIZED = 3, BLE_STATE_PO
/** CBManagerAuthorization.allowedAlways. */
const BLE_AUTHORIZATION_ALLOWED = 3
/**
* Per address, the peripheral identifier and side last heard there and (since 2026-11-16)
* the advertisement itself, from which an arm that has stopped advertising is rebuilt.
*/
const PERIPHERAL_KEY_PREFIX = 'ios.ble.peripheral.'
type RememberedPeripheral = { identifier: string; role: string; name?: string; manufacturerData?: string }
export type BluetoothEvent =
| { kind: 'state'; state: number; authorization: number }
| { kind: 'advertisement'; identifier: string; name: string; manufacturerData: string; rssi: number; connectable: boolean }
@@ -28,6 +38,8 @@ export class IosBluetooth {
private readonly native = FaceclawKitIosBleCentral.alloc().initWithPlatform(FaceclawKitIosProtocolPlatform.shared)
private readonly listeners = new Set<(event: BluetoothEvent) => void>()
readonly devices = new Map<string, IosDevice>()
/** The record last written per settings key, so repeated advertisements don't rewrite it. */
private readonly rememberedValues = new Map<string, string>()
private scanTimer: ReturnType<typeof setTimeout> | null = null
private scanGeneration = 0
private readonly scanListener: object
@@ -47,12 +59,68 @@ export class IosBluetooth {
const device = identifyIosPeripheral(raw)
if (device) {
this.devices.set(device.address, device)
ApplicationSettings.setString(`ios.ble.peripheral.${device.address}`, JSON.stringify({ identifier: device.identifier, role: device.role }))
this.remember(device)
this.emit({ kind: 'device', device })
}
},
})
}
/** Persists what `device` advertised; skips the write when nothing changed (scans report duplicates). */
private remember(device: IosDevice): void {
const key = `${PERIPHERAL_KEY_PREFIX}${device.address}`
const record: RememberedPeripheral = { identifier: device.identifier, role: device.role, name: device.name, manufacturerData: device.manufacturerData }
const value = JSON.stringify(record)
if (this.rememberedValues.get(key) === value) return
this.rememberedValues.set(key, value)
ApplicationSettings.setString(key, value)
}
/**
* Even arms the system is connected to, rebuilt from what they last advertised here. A
* connected arm doesn't advertise, and iOS keeps the right arm's ANCS link open after a
* session ends (even after pairing other glasses), so a scan alone hears only the left
* arm. An arm this phone has never heard advertise can't be rebuilt: its MAC and the
* pair's serial come only from the advertisement. The device has no signal strength.
*/
systemConnectedDevices(): IosDevice[] {
if (this.state !== BLE_STATE_POWERED_ON) return []
let connected: any
try {
connected = this.native.systemConnectedG2Arms()
} catch (error) {
// Scanning still works without this; don't let it take the pairing page down.
console.log(`[IosBluetooth] systemConnectedG2Arms failed: ${error}`)
return []
}
if (!connected?.count) return []
const records = new Map<string, { address: string; record: RememberedPeripheral }>()
for (const key of ApplicationSettings.getAllKeys()) {
if (!key.startsWith(PERIPHERAL_KEY_PREFIX)) continue
try {
const record = JSON.parse(ApplicationSettings.getString(key, '{}')) as RememberedPeripheral
if (typeof record.identifier === 'string') records.set(record.identifier.toUpperCase(), { address: key.slice(PERIPHERAL_KEY_PREFIX.length), record })
} catch { /* Malformed: a fresh advertisement rewrites it. */ }
}
const identity = loadPairedGlassesIdentity()
const devices: IosDevice[] = []
for (let i = 0; i < connected.count; i++) {
const peripheral = connected.objectAtIndex(i)
const identifier = String(peripheral.identifier)
const remembered = records.get(identifier)
if (!remembered) continue
const { address, record } = remembered
let name = record.name ?? ''
let manufacturerData = record.manufacturerData ?? ''
if (!manufacturerData && identity && (address === identity.leftAddress || address === identity.rightAddress)) {
// Remembered before advertisements were: the saved pairing supplies the serial.
const side = address === identity.leftAddress ? 'left' : 'right'
name ||= side === 'left' ? identity.leftName : identity.rightName
manufacturerData = g2ManufacturerData(serialKey(identity.serial) ?? '', address) ?? ''
}
const device = identifyIosPeripheral({ identifier, name: name || String(peripheral.name ?? ''), manufacturerData, rssi: 0, connectable: true })
if (device?.address === address && device.role === record.role) devices.push({ ...device, rssi: null, source: 'paired' })
}
return devices
}
onEvent(listener: (event: BluetoothEvent) => void): () => void {
this.listeners.add(listener); return () => { this.listeners.delete(listener) }
}
@@ -111,7 +179,7 @@ export class IosBluetooth {
const live = this.devices.get(address)
if (live && live.role !== role) throw new Error(`${address} advertises as ${live.role}, not ${role}. Check Configure devices.`)
try {
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
if (saved.role === role && typeof saved.identifier === 'string') result[role] = saved.identifier
} catch { /* A fresh scan can recover malformed settings. */ }
}
@@ -138,13 +206,15 @@ export class IosBluetooth {
/** The peripheral last heard at this address, if it was on this side. */
peripheralIdentifier(address: string, role: string): string | null {
try {
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
return saved.role === role && typeof saved.identifier === 'string' ? saved.identifier : null
} catch {
return null
}
}
forget(address: string): void { ApplicationSettings.remove(`ios.ble.peripheral.${address}`); this.devices.delete(address) }
forget(address: string): void {
ApplicationSettings.remove(`${PERIPHERAL_KEY_PREFIX}${address}`); this.rememberedValues.delete(`${PERIPHERAL_KEY_PREFIX}${address}`); this.devices.delete(address)
}
}
let instance: IosBluetooth | null = null
export function iosBluetooth(): IosBluetooth { return instance ??= new IosBluetooth() }
+24
View File
@@ -0,0 +1,24 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/** The iOS twin of keyboard-link.ts, over the running shared-Kotlin session. */
export function keyboardLinkSession(): unknown {
return FaceclawCommunicatorBridge.getActive();
}
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
return (session as FaceclawCommunicatorBridge).sendFaceclawKeyboardControl(record, label);
}
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
const unsubscribe = (session as FaceclawCommunicatorBridge).addKeyboardEventListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a reconnect.
}
};
}
+40
View File
@@ -0,0 +1,40 @@
import { toJavaBytes } from "./cloud-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
/**
* Platform plumbing for the CFW keyboard bridge (app/g2/keyboard/): the running
* session to send field-130 controls on and to hear field-131 records from.
* The session is opaque and is a different object after a reconnect.
*/
export function keyboardLinkSession(): unknown {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
return (session as any).sendFaceclawKeyboardControl(toJavaBytes(record), label) !== false;
}
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
const communicator = session as any;
const proxy = new com.faceclaw.app.FaceclawKeyboardEventListener({
onKeyboardEvent: (body: any) => listener(toUint8Array(body)),
});
communicator.addKeyboardEventListener(proxy);
return () => {
try {
communicator.removeKeyboardEventListener(proxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
};
}
+50
View File
@@ -0,0 +1,50 @@
import { knownFolders, path as paths } from '@nativescript/core'
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
declare const FaceclawKitIosMicRecordings: any
/**
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
* Documents/Downloads while it is captured, through the shared Kotlin writer
* that Android's captures are recorded with. The file is written about a
* second at a time, so a crash leaves everything up to the last second.
*/
export class MicRecorder {
private constructor(private recorder: any, readonly path: string) {}
/**
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
* `info` describes the capture and is stored in the file.
*/
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.mka`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
return recorder ? new MicRecorder(recorder, path) : null
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.recorder || !pcm.length) return
// The handle is the buffer's start: a view into a larger buffer is copied out first.
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
try {
// Copied by the recorder before this returns.
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
} catch {
// Stop recording rather than retry.
this.close()
}
}
close(): void {
try { this.recorder?.finish() } catch { /* already closed */ }
this.recorder = null
}
}
+29
View File
@@ -0,0 +1,29 @@
declare const android: any;
declare const java: any;
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
export const RECORDING_EXTENSION = ".mka";
export function isMicRecordingFile(name: string): boolean {
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
}
/**
* The path for a new recording in the public Downloads directory:
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
* Downloads isn't available. The file itself is created by whoever records.
*/
export function newRecordingPath(name: string): string | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
return String(file.getAbsolutePath());
} catch (error) {
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
return null;
}
}
+153
View File
@@ -0,0 +1,153 @@
/**
* Platform-independent half of the SSH transport bindings (ssh.ts /
* ssh.ios.ts): the types the app uses and the decoding of the Go
* transport's events (native/ssh/faceclawssh.go, Listener).
*/
/** Where and as whom to connect. */
export type SshEndpoint = {
host: string;
port: number;
user: string;
/** Pinned host keys (SSH wire format, base64). Empty: first connection reports hostkey-unknown. */
hostKeys: string[];
/** Saved password, or "" to try only the device key. */
password: string;
/** Offer the device's client key. */
useKey: boolean;
};
export type SshChannelSpec =
| {
/** Run a command on the host; text is exchanged a line at a time. */
type: "exec";
command: string;
}
| {
/** WebSocket to a URL resolved on the host's side (e.g. ws://127.0.0.1:8737/). */
type: "websocket";
url: string;
};
export type SshHostKey = {
/** SSH key type, e.g. "ssh-ed25519". */
type: string;
/** OpenSSH-style "SHA256:..." fingerprint. */
fingerprint: string;
/** SSH wire format, base64: the value to pin in SshEndpoint.hostKeys. */
key: string;
};
export type SshFailureKind =
| "hostkey-unknown"
| "hostkey-mismatch"
| "auth"
| "network"
| "channel"
| "config"
| "internal";
export type SshFailure = {
kind: SshFailureKind;
message: string;
/** For hostkey-*: the key the host presented. */
hostKey?: SshHostKey;
/** For auth: the host takes passwords / we sent one / we offered our key. */
passwordAllowed?: boolean;
passwordTried?: boolean;
keyTried?: boolean;
};
export type SshClosed = {
/** Exec channels: the command's exit status (and signal, if killed). */
exitStatus?: number;
signal?: string;
/** WebSocket channels: the close frame's code and reason. */
code?: number;
reason?: string;
};
export type SshChannelListener = {
onOpen(): void;
onText(text: string): void;
onStderr?(line: string): void;
/** Terminal: the channel ended normally. */
onClosed(info: SshClosed): void;
/** Terminal: the channel couldn't open or was lost. */
onFailure(failure: SshFailure): void;
};
export type SshChannel = {
/** Queue a websocket text frame / an exec stdin line. False once closed. */
send(text: string): boolean;
/** Flush queued sends, then end the channel. No events follow. */
close(): void;
};
export type SshClientKey = {
/** "ecdsa-sha2-nistp256 AAAA... comment", for ~/.ssh/authorized_keys. */
authorizedKey: string;
fingerprint: string;
};
/** Build the Go transport's openSpec JSON. */
export function encodeSshSpec(endpoint: SshEndpoint, channel: SshChannelSpec): string {
return JSON.stringify({
host: endpoint.host,
port: endpoint.port,
user: endpoint.user,
password: endpoint.password,
hostKeys: endpoint.hostKeys,
useKey: endpoint.useKey,
channel,
});
}
/** Route one transport event to the listener. Unknown kinds are ignored. */
export function dispatchSshEvent(listener: SshChannelListener, kind: string, payload: string): void {
switch (kind) {
case "open":
listener.onOpen();
return;
case "text":
listener.onText(payload);
return;
case "stderr":
listener.onStderr?.(payload);
return;
case "closed":
listener.onClosed(parseJson(payload) ?? {});
return;
case "failure": {
const failure = parseJson(payload);
listener.onFailure(
failure && typeof failure.kind === "string"
? failure
: { kind: "internal", message: `bad failure payload: ${payload}` },
);
return;
}
}
}
function parseJson(text: string): any {
try {
return JSON.parse(text);
} catch {
return null;
}
}
/** Human summary of a failure for status lines. */
export function describeSshFailure(failure: SshFailure): string {
switch (failure.kind) {
case "hostkey-unknown":
return "Host key not yet trusted.";
case "hostkey-mismatch":
return "HOST KEY CHANGED; refusing to connect.";
case "auth":
return failure.message || "Authentication failed.";
default:
return failure.message || failure.kind;
}
}
+69
View File
@@ -0,0 +1,69 @@
import {
dispatchSshEvent,
encodeSshSpec,
type SshChannel,
type SshChannelListener,
type SshChannelSpec,
type SshClientKey,
type SshEndpoint,
} from "./ssh-common";
export * from "./ssh-common";
declare const FaceclawSshChannel: any;
declare const FaceclawSshKeys: any;
// Like socket.ios.ts: Go delivers events on its own threads, so the native
// side queues them and this isolate drains the queue on a timer.
const POLL_INTERVAL_MS = 25;
/** iOS SSH transport: FaceclawSshChannel (ObjC) over the gomobile-bound Go package in native/ssh. */
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
const native = FaceclawSshChannel.alloc().initWithSpecUseKey(encodeSshSpec(endpoint, channel), endpoint.useKey);
let ended = false;
const stop = () => {
if (ended) return;
ended = true;
clearInterval(timer);
native.close();
};
const timer = setInterval(() => {
const events = JSON.parse(String(native.takeEvents())) as Array<{ kind: string; payload: string }>;
for (const event of events) {
if (ended) break;
if (event.kind === "closed" || event.kind === "failure") {
// Terminal events: stop polling, then deliver.
ended = true;
clearInterval(timer);
}
dispatchSshEvent(listener, event.kind, event.payload);
}
}, POLL_INTERVAL_MS);
return {
send: (text) => !ended && native.send(text) === true,
close: stop,
};
}
export function sshClientKey(comment: string): SshClientKey {
const authorizedKey = FaceclawSshKeys.authorizedKeyWithComment(comment);
const fingerprint = FaceclawSshKeys.fingerprint();
if (!authorizedKey || !fingerprint) throw new Error("Could not create the SSH client key.");
return { authorizedKey: String(authorizedKey), fingerprint: String(fingerprint) };
}
export function getSshSecret(id: string): string {
return String(FaceclawSshKeys.secretForId(id) ?? "");
}
export function setSshSecret(id: string, value: string): void {
FaceclawSshKeys.setSecretForId(value, id);
}
export function removeSshSecret(id: string): void {
FaceclawSshKeys.removeSecretForId(id);
}
export function copyToPhoneClipboard(_label: string, text: string): void {
FaceclawSshKeys.copyToClipboard(text);
}
+66
View File
@@ -0,0 +1,66 @@
import { Utils } from "@nativescript/core";
import {
dispatchSshEvent,
encodeSshSpec,
type SshChannel,
type SshChannelListener,
type SshChannelSpec,
type SshClientKey,
type SshEndpoint,
} from "./ssh-common";
export * from "./ssh-common";
declare const com: any;
/**
* Android SSH transport: FaceclawSshChannel (Kotlin) over the gomobile-bound
* Go package in native/ssh. Callable from the main isolate or a worker;
* events arrive on the calling isolate's thread.
*/
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
let closed = false;
const proxy = new com.faceclaw.app.FaceclawSshListener({
onEvent: (kind: string, payload: string) => {
if (closed) return;
dispatchSshEvent(listener, String(kind), payload == null ? "" : String(payload));
},
});
const native = new com.faceclaw.app.FaceclawSshChannel(encodeSshSpec(endpoint, channel), endpoint.useKey, proxy);
return {
send: (text) => !closed && native.send(text) === true,
close: () => {
if (closed) return;
closed = true;
native.close();
},
};
}
function context(): any {
return Utils.android.getApplicationContext();
}
/** The device's SSH client key (generated on first use). */
export function sshClientKey(comment: string): SshClientKey {
return {
authorizedKey: String(com.faceclaw.app.FaceclawSshKeys.authorizedKey(comment)),
fingerprint: String(com.faceclaw.app.FaceclawSshKeys.fingerprint()),
};
}
export function getSshSecret(id: string): string {
return String(com.faceclaw.app.FaceclawSshKeys.getSecret(context(), id) ?? "");
}
export function setSshSecret(id: string, value: string): void {
com.faceclaw.app.FaceclawSshKeys.setSecret(context(), id, value);
}
export function removeSshSecret(id: string): void {
com.faceclaw.app.FaceclawSshKeys.removeSecret(context(), id);
}
export function copyToPhoneClipboard(label: string, text: string): void {
com.faceclaw.app.FaceclawSshKeys.copyToClipboard(context(), label, text);
}
+59
View File
@@ -0,0 +1,59 @@
/**
* Session-list and connection-state types shared by the Terminal app's
* backends (g2mirror-client.ts, apps/terminal/tmux-client.ts). Kept free of
* platform imports so the tmux client can be unit-tested under node.
*/
export type G2MirrorSession = {
socket: string;
pid: number;
cwdHint: string;
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
lastBellAt: number | null;
/**
* Unix epoch ms (host clock) of the app's last output, or null if none
* observed. Fresh to within the wrapper's 2s activity reporting interval
* while we're connected; compare against other host timestamps, not the
* phone clock (see the worker's local-receive-time activity tracking).
*/
lastOutputAt: number | null;
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
title: string | null;
};
export type G2MirrorPhase =
| "idle"
| "connecting"
| "connected" // handshake accepted; can list/attach
| "attached" // relaying to one session
| "failed";
export type G2MirrorState = {
phase: G2MirrorPhase;
status: string;
sessions: G2MirrorSession[];
attachedCommand: string;
/**
* server_name from the init success reply: a human-readable name for the
* machine the server runs on. "" until the handshake succeeds (or when an
* older server omits the field).
*/
serverName: string;
/**
* False when reconnecting can't help until the user changes something on
* the host (e.g. tmux missing or too old). Absent means retryable.
*/
retryable?: boolean;
};
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
export type G2MirrorHistoryReply = {
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
start: number;
/** Oldest index still retained by the wrapper (older requests return nothing). */
oldest: number;
/** Current splice point (next line to be archived). */
next: number;
/** Plain text, ANSI-stripped, oldest-to-newest. */
lines: string[];
};
+8 -5
View File
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
import { WavRecorder } from './wav-recorder.ios'
import { MicRecorder } from './mic-recorder.ios'
import { pcmLevel } from '../util/audio-level'
declare const FaceclawSpeech: any
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
private continuous = false
// Shown instead of the native "ended" status once a drain the bridge started completes.
private drainStatus: string | null = null
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
private recorder: MicRecorder | null = null
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false
private readonly recordingListeners = new Set<() => void>()
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
* until called with another name or ''. Only cloud captures supply PCM.
*/
recordTo(name: string): void {
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
private record(pcm: Uint8Array): void {
if (!this.recording) return
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName)
this.recorder = MicRecorder.open(this.recordingName, {
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
})
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
}
+319 -27
View File
@@ -1,10 +1,10 @@
import type { OpenAiSttModel } from "./openai-stt";
import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { newRecordingPath } from "./mic-recording";
import { SpeechPauseDetector } from "./speech-pause";
import { WavRecorder } from "./wav-recorder";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
@@ -86,6 +86,49 @@ export type PushToTalkOptions = {
* frame). Null/absent leaves the mic omnidirectional.
*/
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
/**
* Capture from the temples' own microphones (the custom firmware's
* mic_control stream) instead of the glasses' mixed stream. Each temple
* streams its microphones on its own link and clock; the phone mixes them
* into the mono feed that is transcribed. Null/absent keeps the mixed
* stream; ignored with usePhoneMic.
*/
micArray?: MicArrayCapture | null;
/**
* Play this recording (a .mka file from Save Recording) in real time in
* place of any microphone. The capture needs no glasses, and ends by
* itself when the recording does.
*/
replay?: { path: string; startMs: number } | null;
/** What the caller knows of the capture, for the description stored in a recording of it. */
captureInfo?: Record<string, unknown>;
};
export type MicArrayCapture = {
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
configure: Uint8Array;
};
/** Progress of a capture that replays a recording; positions are ms into it. */
export type VoiceReplayEvent = {
/**
* `start` once the file is open, `position` about once per second played,
* `seek` when a jump took effect, `end` when the recording ran out, and
* `error` when it could not be opened (the status says why).
*/
kind: "start" | "position" | "seek" | "end" | "error";
positionMs: number;
durationMs: number;
};
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
export type VoiceEventSource = {
onStatus(listener: (state: VoiceControlState) => void): () => void;
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
onSpeechPause(listener: () => void): () => void;
onMicLevel(listener: (level: number) => void): () => void;
readonly recording: boolean;
readonly recordingFailed: boolean;
};
/** Who currently wants the mic running. */
@@ -132,9 +175,13 @@ export class FaceclawVoiceControlBridge {
private captureEndpointing = false;
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
private cloudClient: CloudSttClient | null = null;
// What the running STT stream transcribes with (transcriptionKey); a holder
// asking for something else restarts the stream rather than sharing it.
// What the running STT stream captures from and transcribes with
// (streamKey); a holder asking for something else restarts the stream
// rather than sharing it.
private streamProvider: string | null = null;
// The options of the running stream, for the description of a recording of it.
private streamOptions: PushToTalkOptions | null = null;
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
// controller runs in the decode-only "cloud" mode and every decoded PCM
// frame is broadcast to these listeners. STT capture preempts it — a live
@@ -152,11 +199,14 @@ export class FaceclawVoiceControlBridge {
// the bridge swallow it.
private verificationActive = false;
private verificationRejected = false;
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
// set: every source's chunks as they arrive, before any processing, written
// by the controller. The file outlives a capture, so a restarted one keeps
// adding to it, and it is created by the first audio, so a failed start
// leaves no file.
private recordingName = "";
private recorder: WavRecorder | null = null;
// The file is open in the controller.
private recordingOpen = false;
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false;
private readonly recordingListeners = new Set<() => void>();
@@ -253,16 +303,55 @@ export class FaceclawVoiceControlBridge {
}
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* End the continuous capture as an utterance: stop it and have the
* transcript of its last audio come as a final. For a capture whose audio
* ran out by itself (a replay).
*/
finishContinuousCapture(): void {
this.releaseCapture("continuous", true);
}
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
this.replayListeners.add(listener);
return () => this.replayListeners.delete(listener);
}
/** Continue the running replay from `positionMs` into its recording. */
seekReplay(positionMs: number): void {
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
}
/** Hold the running replay where it is, or let it go on. */
setReplayPaused(paused: boolean): void {
this.controller?.setReplayPaused(paused);
}
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
isCaptureLive(): boolean {
return this.micIsLive();
}
/**
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
* until called with another name or "".
*/
recordTo(name: string): void {
if (name === this.recordingName) return;
this.recorder?.close();
this.recorder = null;
this.closeRecording();
this.setRecording(name, false);
}
private closeRecording(): void {
if (!this.recordingOpen) return;
this.recordingOpen = false;
try {
this.controller?.stopRecording();
} catch (error) {
console.warn(`Voice: closing the recording failed: ${error}`);
}
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean {
return this.recordingName !== "" && !this.recordingFailure;
@@ -288,18 +377,48 @@ export class FaceclawVoiceControlBridge {
for (const listener of Array.from(this.recordingListeners)) listener();
}
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
private record(pcm: any): void {
if (!this.recording) return;
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName);
if (!this.recorder) {
this.setRecording(this.recordingName, true);
return;
}
console.log(`Voice: recording audio to ${this.recorder.path}`);
/**
* Audio arrived for a continuous capture: open the recording asked for, if
* it isn't open yet. The controller writes it from then on, as the audio
* arrives on its own thread.
*/
private ensureRecording(): void {
if (!this.recording || this.recordingOpen) return;
const path = newRecordingPath(this.recordingName);
let opened = false;
try {
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
} catch (error) {
console.warn(`Voice: could not create the recording: ${error}`);
}
this.recorder.append(pcm);
if (!opened) {
this.setRecording(this.recordingName, true);
return;
}
this.recordingOpen = true;
console.log(`Voice: recording audio to ${path}`);
}
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
private syncRecordingEnabled(): void {
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
}
/** The running capture, as the JSON stored in a recording of it. */
private describeCapture(): Record<string, unknown> {
const options = this.streamOptions;
return {
recording: this.recordingName,
startedAt: new Date().toISOString(),
source: options?.usePhoneMic ? "phone" : "glasses",
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
processing: {
noiseSuppression: Boolean(options?.noiseSuppression),
beamFilter: options?.beamFilter ?? null,
},
...options?.captureInfo,
};
}
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
@@ -366,6 +485,8 @@ export class FaceclawVoiceControlBridge {
// preview-mode capture may have left on the shared controller.
this.controller?.setUsePhoneMic(false);
this.controller?.setEndpointing(false);
this.controller?.setMicArray(null);
this.controller?.setReplay(null, 0);
// Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing.
this.controller?.setNoiseSuppression(false);
@@ -401,18 +522,33 @@ export class FaceclawVoiceControlBridge {
const micLive = this.micIsLive();
this.suspendedHolders.delete(holder);
this.captureHolders.add(holder);
if (micLive && this.streamProvider === transcriptionKey(options)) {
if (micLive && this.streamProvider === streamKey(options)) {
// Mic already running; the new holder just shares the existing stream
// (transcripts are already broadcast to its listeners).
this.syncRecordingEnabled();
return;
}
// Any leftover capture is dead, or transcribes with another provider than
// Any leftover capture is dead, or captures or transcribes otherwise than
// this holder needs; drop it before starting the new one.
if (this.started) this.teardownCapture();
this.streamProvider = transcriptionKey(options);
this.streamProvider = streamKey(options);
this.streamOptions = options;
this.ensureController();
this.syncRecordingEnabled();
// A recording under way carries on into this capture: note what changed.
if (this.recordingOpen) {
try {
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
} catch (error) {
console.warn(`Voice: recording event failed: ${error}`);
}
}
this.controller?.setCommunicator(options.communicator);
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
this.controller?.setMicArray(
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
);
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
this.controller?.setEndpointing(Boolean(options.endpointing));
this.captureEndpointing = Boolean(options.endpointing);
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
@@ -437,7 +573,7 @@ export class FaceclawVoiceControlBridge {
this.speechPause.reset();
// Replaces whatever the last capture left behind ("Listening...", an
// error, the permission prompt) until the controller reports progress.
this.setStatus("Starting microphone...");
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
@@ -485,6 +621,7 @@ export class FaceclawVoiceControlBridge {
if (commit) this.cloudClient?.finish();
return;
}
this.syncRecordingEnabled();
if (this.captureHolders.size > 0) {
// Another holder still wants the mic; keep it running.
return;
@@ -509,6 +646,7 @@ export class FaceclawVoiceControlBridge {
this.captureHolders.clear();
this.suspendedHolders.clear();
this.suspendedRaw = false;
this.syncRecordingEnabled();
this.teardownCapture();
this.setStatus("Voice control stopped.");
}
@@ -526,6 +664,7 @@ export class FaceclawVoiceControlBridge {
this.suspendedHolders.add(holder);
}
this.captureHolders.clear();
this.syncRecordingEnabled();
this.suspendedRaw = this.rawActive;
this.teardownCapture();
if (this.suspendedHolders.size > 0) {
@@ -583,6 +722,7 @@ export class FaceclawVoiceControlBridge {
private teardownCapture(): void {
this.rawActive = false;
this.streamProvider = null;
this.streamOptions = null;
if (global.isAndroid) {
this.controller?.stop();
}
@@ -616,7 +756,7 @@ export class FaceclawVoiceControlBridge {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
// Push-to-talk on a capture of its own is not recorded.
if (this.captureHolders.has("continuous")) this.record(pcm);
if (this.captureHolders.has("continuous")) this.ensureRecording();
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
@@ -647,6 +787,10 @@ export class FaceclawVoiceControlBridge {
);
}
},
onReplay: (kind: string, positionMs: number, durationMs: number) => {
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
for (const listener of Array.from(this.replayListeners)) listener(event);
},
});
this.controller.setListener(this.listenerProxy);
}
@@ -683,6 +827,21 @@ export class FaceclawVoiceControlBridge {
}
}
/**
* What a capture reads and transcribes with. A running capture is shared only
* by holders asking for the same: the provider and language, the microphones
* (the mixed stream, or the temples' own under a given configuration), or the
* recording it replays.
*/
function streamKey(options: PushToTalkOptions): string {
const source = options.replay
? `replay:${options.replay.path}`
: options.micArray && !options.usePhoneMic
? `array:${Array.from(options.micArray.configure).join(",")}`
: "mixed";
return `${transcriptionKey(options)}|${source}`;
}
function toJavaFloats(values: Float32Array): any {
const javaFloats = Array.create("float", values.length);
for (let i = 0; i < values.length; i++) {
@@ -692,3 +851,136 @@ function toJavaFloats(values: Float32Array): any {
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
/**
* A recording played back as a capture of its own: transcribed by the same
* pipeline as the microphone, with the same settings, on a bridge and native
* controller separate from the microphone's. Nothing here touches the mic,
* so voice input and a live Transcribe window carry on beside a replay.
*/
export class VoiceReplay {
private readonly bridge = new FaceclawVoiceControlBridge();
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
private readonly offReplay: () => void;
private position = 0;
private duration = 0;
private current: VoiceReplayState = "opening";
private closed = false;
/**
* `options` gives the capture options (provider, keys, processing) each
* time the replay starts, so a restart picks up changed settings.
*/
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
}
/** The replay's status, transcript and level events. */
get events(): VoiceEventSource {
return this.bridge;
}
/** Milliseconds into the recording. */
get positionMs(): number {
return this.position;
}
/** The recording's length in ms; 0 until it has been opened. */
get durationMs(): number {
return this.duration;
}
get state(): VoiceReplayState {
return this.current;
}
/**
* The position, duration or state changed. `jumpedToMs` is given when
* playback moved there other than by playing on: a jump, or a start
* part-way in. It comes after the transcript of what played before.
*/
onChange(listener: (jumpedToMs?: number) => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
/** Play from `positionMs` into the recording. */
start(positionMs = 0): void {
if (this.closed) return;
this.position = Math.max(0, positionMs);
this.set("opening");
this.bridge.startContinuousCapture({
...this.options(),
communicator: null,
usePhoneMic: false,
micArray: null,
replay: { path: this.path, startMs: this.position },
});
}
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
jumpTo(positionMs: number): void {
if (this.closed) return;
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
this.bridge.seekReplay(target);
this.position = target;
this.notify();
return;
}
this.start(target);
}
setPaused(paused: boolean): void {
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
this.bridge.setReplayPaused(paused);
this.set(paused ? "paused" : "playing");
}
/** Stop for good and release the capture. */
close(): void {
if (this.closed) return;
this.closed = true;
this.offReplay();
this.listeners.clear();
this.bridge.stop();
}
private handle(event: VoiceReplayEvent): void {
this.duration = event.durationMs;
switch (event.kind) {
case "start":
this.position = event.positionMs;
this.current = "playing";
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
return;
case "seek":
this.position = event.positionMs;
this.notify(event.positionMs);
return;
case "end":
this.position = event.durationMs;
// Its last audio is still to be transcribed (a cloud provider's final).
this.bridge.finishContinuousCapture();
this.set("ended");
return;
case "error":
this.set("failed");
return;
default:
this.position = event.positionMs;
this.notify();
}
}
private set(state: VoiceReplayState): void {
this.current = state;
this.notify();
}
private notify(jumpedToMs?: number): void {
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
}
}
-22
View File
@@ -1,22 +0,0 @@
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
export function wavHeader(dataBytes: number): Uint8Array {
const header = new Uint8Array(44);
const view = new DataView(header.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + dataBytes, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, 16000, true);
view.setUint32(28, 32000, true); // byte rate
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
ascii(36, "data");
view.setUint32(40, dataBytes, true);
return header;
}
-75
View File
@@ -1,75 +0,0 @@
import { knownFolders, path as paths } from '@nativescript/core'
import { wavHeader } from './wav-header'
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
const FLUSH_BYTES = 32000
/**
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private pending: Uint8Array[] = []
private pendingBytes = 0
private dataBytes = 0
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.wav`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
return new WavRecorder(handle, path)
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.handle || !pcm.length) return
this.pending.push(pcm.slice())
this.pendingBytes += pcm.length
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
}
close(): void {
this.flush()
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
this.handle = null
}
private flush(): void {
const handle = this.handle
if (!handle || !this.pendingBytes) return
const chunk = new Uint8Array(this.pendingBytes)
let offset = 0
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
this.pending = []
this.pendingBytes = 0
try {
// NativeScript throws a failed call's NSError (storage full, file removed).
handle.writeDataError(toData(chunk))
handle.seekToOffsetError(0)
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
handle.seekToEndReturningOffsetError(null)
this.dataBytes += chunk.length
return
} catch {
// Stop recording rather than retry.
}
try { handle.closeAndReturnError() } catch { /* ignore */ }
this.handle = null
}
}
function toData(bytes: Uint8Array): NSData {
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
}
-42
View File
@@ -1,42 +0,0 @@
declare const android: any;
declare const java: any;
declare const com: any;
const SAMPLE_RATE = 16000;
/**
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
* native recorder keeps the header's sizes current, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private constructor(private recorder: any, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}.wav`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
const path = String(file.getAbsolutePath());
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
} catch (error) {
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
return null;
}
}
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
append(pcm: any): void {
this.recorder?.append(pcm);
}
close(): void {
this.recorder?.finish();
this.recorder = null;
}
}