Port developer debug tests to iOS

This commit is contained in:
James Babcock committed 2026-10-31 13:57:18 -07:00
1 parent bc57557f04
commit a5eda29368
16 files changed
+737 -151

No files matched your search

-8
View File
@@ -50,32 +50,24 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
},
{
label: "Buzzer demo",
disabled: global.isIOS,
description: global.isIOS ? "Glasses sound playback is not available on iOS yet." : undefined,
onSelect: (ctx) => {
ctx.stack.push(new BuzzerDemoLayer());
},
},
{
label: "Accelerometer demo",
disabled: global.isIOS,
description: global.isIOS ? "Not available on iOS yet." : undefined,
onSelect: (ctx) => {
ctx.stack.push(new AccelerometerDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "Light sensor",
disabled: global.isIOS,
description: global.isIOS ? "Not available on iOS yet." : undefined,
onSelect: (ctx) => {
ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "Microphones",
disabled: global.isIOS,
description: global.isIOS ? "Not available on iOS yet." : undefined,
onSelect: openMicrophones,
},
{
+18 -41
View File
@@ -1,7 +1,6 @@
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import { useMicControlSetting } from "../../ui/dashboard-settings";
import { toJavaBytes } from "../../native/cloud-stt";
import { toUint8Array } from "../../util/array-util";
import { addMicStatusListener, micControlSession, sendMicControlRecord } from "../../native/mic-control-link";
import {
MIC_LEASE_MS,
decodeMicStatus,
@@ -14,8 +13,6 @@ import {
type MicStatus,
} from "./mic-protocol";
declare const com: any;
/**
* Host-side controller for the CFW mic_control array: sends the same config
* to both temples (each temple is its own endpoint and its own 2-mic array),
@@ -30,15 +27,6 @@ const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/**
* True when the "Use microphone control" developer setting is on. With the
* setting off, Faceclaw behaves exactly as it would on firmware without the
@@ -71,9 +59,9 @@ export function saveMicConfig(config: MicConfig): void {
class MicArrayController {
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
private statuses: MicArrayStatuses = { left: null, right: null };
// The Java listener proxy must stay referenced or it gets GC'd.
private statusProxy: any = null;
private proxyCommunicator: any = null;
// Holds the native listener proxy, which must stay referenced or it gets GC'd.
private unsubscribeStatus: (() => void) | null = null;
private statusSession: unknown = null;
private leaseTimer: ReturnType<typeof setInterval> | null = null;
private armed = false;
@@ -158,39 +146,28 @@ class MicArrayController {
rightTemple = true,
leftTemple = true,
): boolean {
const communicator = activeCommunicator();
if (!communicator) return false;
const session = micControlSession();
if (!session) return false;
try {
this.ensureStatusProxy(communicator);
const record = encodeMicControl(op, config);
// False when the session wasn't ready and nothing was queued.
return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
this.ensureStatusListener(session);
return sendMicControlRecord(session, encodeMicControl(op, config), label, rightTemple, leftTemple);
} catch (error) {
console.warn(`mic control ${label} failed: ${error}`);
return false;
}
}
private ensureStatusProxy(communicator: any): void {
if (this.statusProxy && this.proxyCommunicator === communicator) return;
if (this.statusProxy && this.proxyCommunicator) {
try {
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
}
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
onMicStatus: (body: any, arm: string) => {
const status = decodeMicStatus(toUint8Array(body));
if (!status) return;
const side: MicSide = String(arm) === "L" ? "left" : "right";
this.statuses = { ...this.statuses, [side]: { ...status, side } };
this.statusListeners.forEach((listener) => listener(this.statuses));
},
private ensureStatusListener(session: unknown): void {
if (this.unsubscribeStatus && this.statusSession === session) return;
this.unsubscribeStatus?.();
this.statusSession = session;
this.unsubscribeStatus = addMicStatusListener(session, (body, arm) => {
const status = decodeMicStatus(body);
if (!status) return;
const side: MicSide = arm === "L" ? "left" : "right";
this.statuses = { ...this.statuses, [side]: { ...status, side } };
this.statusListeners.forEach((listener) => listener(this.statuses));
});
this.proxyCommunicator = communicator;
communicator.addMicStatusListener(this.statusProxy);
}
private syncLeaseTimer(): void {
+93
View File
@@ -0,0 +1,93 @@
/**
* Platform-neutral ambient-light types and the CFW report decoder (settings
* field 105), shared by ambient-light.ts and ambient-light.ios.ts.
*/
export const ALS_REPORT_VERSION = 1;
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
export type AmbientLightReport = {
reason: AmbientLightReason;
/** The sensor driver is open (readings are live). */
opened: boolean;
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
passive: boolean;
/** The user's stock auto-brightness setting is on. */
autoBrightness: boolean;
/** The last passive read succeeded (only meaningful while passive). */
readOk: boolean;
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
status: number;
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
alsValue: number;
/** Max of the last five readings (what the stock curve is applied to). */
peak: number;
/** Brightness level (0..100) the stock curve would choose for `peak`. */
stockTarget: number;
/** Current brightness setting (0..100). */
brightness: number;
/** Row of the stock brightness curve (0..5) for `peak`. */
gear: number;
/** Stock learned scale factor, Q10 (1024 = 1.0). */
scaleQ10: number;
/** Firmware millisecond tick when the report was built. */
tickMs: number;
};
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
function rd16(b: Uint8Array, i: number): number {
return b[i] | (b[i + 1] << 8);
}
function rd32(b: Uint8Array, i: number): number {
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
}
/** Decode a field-105 record body, or null when it isn't one. */
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
return null;
}
const flags = body[4];
return {
reason: REASONS[body[3]] ?? "poll",
opened: (flags & 0x01) !== 0,
passive: (flags & 0x02) !== 0,
autoBrightness: (flags & 0x04) !== 0,
readOk: (flags & 0x08) !== 0,
status: body[5],
alsValue: rd32(body, 6),
peak: rd32(body, 10),
stockTarget: rd16(body, 14),
brightness: body[16],
gear: body[17],
scaleQ10: rd16(body, 18),
tickMs: rd32(body, 20),
};
}
export type AmbientLightPollingOptions = {
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
intervalMs?: number;
/** Report when the reading moved by at least this much. Default 1. */
minDelta?: number;
/** Also report after this many ms without one (0 = never). Default 5000. */
heartbeatMs?: number;
/**
* Stop automatically when the Faceclaw framebuffer lease is released or
* lapses (the phone went away). Default true.
*/
bindToLease?: boolean;
};
/** The options with their defaults filled in. */
export function resolvePollingOptions(options: AmbientLightPollingOptions): Required<AmbientLightPollingOptions> {
return {
intervalMs: options.intervalMs ?? 500,
minDelta: options.minDelta ?? 1,
heartbeatMs: options.heartbeatMs ?? 5000,
bindToLease: options.bindToLease ?? true,
};
}
+51
View File
@@ -0,0 +1,51 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
import {
decodeAmbientLightReport,
resolvePollingOptions,
type AmbientLightPollingOptions,
type AmbientLightReport,
} from "./ambient-light-types";
export * from "./ambient-light-types";
/**
* Ambient light sensor access on iOS: the same API as ambient-light.ts (see
* there for the CFW query and passive-polling semantics) over the running
* shared-Kotlin session. No-ops while no session is running.
*/
/** Ask the firmware for one ambient-light report (delivered to listeners). */
export function queryAmbientLight(): void {
try {
FaceclawCommunicatorBridge.getActive()?.queryAmbientLight();
} catch (error) {
console.warn(`queryAmbientLight failed: ${error}`);
}
}
/** Start or stop CFW passive light-sensor polling. */
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
try {
FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
} catch (error) {
console.warn(`setAmbientLightPolling failed: ${error}`);
}
}
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
const active = FaceclawCommunicatorBridge.getActive();
if (!active) return () => {};
const unsubscribe = active.addAmbientLightListener((body) => {
const report = decodeAmbientLightReport(body);
if (report) listener(report);
});
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a disconnect.
}
};
}
+9 -85
View File
@@ -16,88 +16,17 @@
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
*/
import { toUint8Array } from "../util/array-util";
import {
decodeAmbientLightReport,
resolvePollingOptions,
type AmbientLightPollingOptions,
type AmbientLightReport,
} from "./ambient-light-types";
declare const com: any;
declare const global: any;
export const ALS_REPORT_VERSION = 1;
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
export type AmbientLightReport = {
reason: AmbientLightReason;
/** The sensor driver is open (readings are live). */
opened: boolean;
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
passive: boolean;
/** The user's stock auto-brightness setting is on. */
autoBrightness: boolean;
/** The last passive read succeeded (only meaningful while passive). */
readOk: boolean;
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
status: number;
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
alsValue: number;
/** Max of the last five readings (what the stock curve is applied to). */
peak: number;
/** Brightness level (0..100) the stock curve would choose for `peak`. */
stockTarget: number;
/** Current brightness setting (0..100). */
brightness: number;
/** Row of the stock brightness curve (0..5) for `peak`. */
gear: number;
/** Stock learned scale factor, Q10 (1024 = 1.0). */
scaleQ10: number;
/** Firmware millisecond tick when the report was built. */
tickMs: number;
};
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
function rd16(b: Uint8Array, i: number): number {
return b[i] | (b[i + 1] << 8);
}
function rd32(b: Uint8Array, i: number): number {
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
}
/** Decode a field-105 record body, or null when it isn't one. */
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
return null;
}
const flags = body[4];
return {
reason: REASONS[body[3]] ?? "poll",
opened: (flags & 0x01) !== 0,
passive: (flags & 0x02) !== 0,
autoBrightness: (flags & 0x04) !== 0,
readOk: (flags & 0x08) !== 0,
status: body[5],
alsValue: rd32(body, 6),
peak: rd32(body, 10),
stockTarget: rd16(body, 14),
brightness: body[16],
gear: body[17],
scaleQ10: rd16(body, 18),
tickMs: rd32(body, 20),
};
}
export type AmbientLightPollingOptions = {
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
intervalMs?: number;
/** Report when the reading moved by at least this much. Default 1. */
minDelta?: number;
/** Also report after this many ms without one (0 = never). Default 5000. */
heartbeatMs?: number;
/**
* Stop automatically when the Faceclaw framebuffer lease is released or
* lapses (the phone went away). Default true.
*/
bindToLease?: boolean;
};
export * from "./ambient-light-types";
function activeCommunicator(): any {
if (!global.isAndroid) return null;
@@ -123,14 +52,9 @@ export function queryAmbientLight(): void {
* on every session start.
*/
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
try {
activeCommunicator()?.setAmbientLightPolling(
enabled,
options.intervalMs ?? 500,
options.minDelta ?? 1,
options.heartbeatMs ?? 5000,
options.bindToLease ?? true,
);
activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
} catch (error) {
console.warn(`setAmbientLightPolling failed: ${error}`);
}
+50
View File
@@ -0,0 +1,50 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/**
* Passive view of the glasses' audio traffic on iOS: the same API as
* audio-monitor.ts over the running shared-Kotlin session. Nothing here sends
* anything to the glasses.
*/
export type AudioArm = "L" | "R" | "?";
/** The session listeners attach to; changes across reconnects. */
export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null {
return FaceclawCommunicatorBridge.getActive();
}
/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */
export function isFaceclawAudioCaptureActive(): boolean {
try {
return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive());
} catch {
return false;
}
}
/**
* Subscribe `communicator` to every audio-characteristic packet and every
* field-104 mic status record body (undecoded). Delivered on the main thread.
* Returns an unsubscribe function.
*/
export function addAudioMonitorListeners(
communicator: FaceclawCommunicatorBridge | null,
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
onStatus: (body: Uint8Array, arm: AudioArm) => void,
): () => void {
if (!communicator) return () => {};
const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs));
const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm)));
return () => {
try {
offPackets();
offStatus();
} catch {
// The session may have been closed during a disconnect.
}
};
}
function toArm(arm: string): AudioArm {
return arm === "L" || arm === "R" ? arm : "?";
}
+109 -1
View File
@@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input";
import { File, knownFolders, path, type ImageSource } from "@nativescript/core";
import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat";
import { type CompassEvent } from "./compass-types";
import { type ImuReading } from "./imu-types";
import { fromData, toData } from "./kotlin-data";
import { previewPixels } from "./ios-graphics";
import { iosBluetooth } from "./ios-bluetooth";
@@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any;
declare const FaceclawKitIosAncsListener: any;
declare const FaceclawKitIosAudioPacketListener: any;
declare const FaceclawKitFaceclawCompassListener: any;
declare const FaceclawKitFaceclawImuListener: any;
declare const FaceclawKitIosAmbientLightListener: any;
declare const FaceclawKitIosMicStatusListener: any;
export type CommunicatorPhase =
@@ -164,6 +168,29 @@ class CompassListener extends NSObject {
}
}
@NativeClass()
class ImuListener extends NSObject {
static ObjCProtocols = [FaceclawKitFaceclawImuListener];
handler!: (reading: ImuReading) => void;
onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void {
this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) });
}
}
@NativeClass()
class AmbientLightListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosAmbientLightListener];
handler!: (body: Uint8Array) => void;
onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); }
}
@NativeClass()
class MicStatusListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosMicStatusListener];
handler!: (body: Uint8Array, arm: string) => void;
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
}
/**
* The iOS twin of the Android communicator bridge: the same public API over
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
@@ -173,6 +200,8 @@ class CompassListener extends NSObject {
* Android so callers never re-enter the bridge from a Kotlin frame.
*/
export class FaceclawCommunicatorBridge {
// The started, not yet closed bridge (see getActive).
private static active: FaceclawCommunicatorBridge | null = null;
private readonly communicator: any;
private readonly listenerProxy: SessionListener;
private ancsProxy: AncsListener | null = null;
@@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge {
private audioListener: AudioPacketListener | null = null;
private microphoneWork: Promise<void> = Promise.resolve();
private microphoneToken = 0;
private closed = false;
/**
* The bridge whose session is running, for modules that reach the glasses
* without the bridge threaded through (imu, ambient-light, audio-monitor):
* the iOS stand-in for Android's FaceclawBleCommunicator.getActive().
* Set by start(), cleared by close().
*/
static getActive(): FaceclawCommunicatorBridge | null {
return FaceclawCommunicatorBridge.active;
}
constructor(addresses: { right: string; left: string; ring?: string }) {
this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress(
@@ -450,7 +490,9 @@ export class FaceclawCommunicatorBridge {
// ----- session ---------------------------------------------------------------------
async start(): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.start(); });
await this.enqueueNativeCall(() => {
if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this;
});
}
async setG2ScreenOn(screenOn: boolean): Promise<void> {
@@ -640,6 +682,8 @@ export class FaceclawCommunicatorBridge {
}
async close(): Promise<void> {
this.closed = true;
if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null;
this.setAncsListeners(false);
this.previewAnimationListeners.clear();
await this.enqueueNativeCall(() => {
@@ -663,6 +707,70 @@ export class FaceclawCommunicatorBridge {
return () => { this.communicator.removeCompassListenerListener(proxy); };
}
// ----- sensors and passive audio, for the developer demos -----------------------------
// Fire-and-forget, so they skip the native-call queue like the compass controls do.
/** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */
setImuReportEnabled(enable: boolean, reportFrq: number): void {
this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq)));
}
/** Accelerometer samples, delivered on the main queue. */
addImuListener(listener: (reading: ImuReading) => void): () => void {
const proxy = ImuListener.new() as ImuListener;
proxy.handler = listener;
this.communicator.addImuListenerListener(proxy);
return () => { this.communicator.removeImuListenerListener(proxy); };
}
/** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */
queryAmbientLight(): void {
this.communicator.queryAmbientLight();
}
setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void {
this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable),
Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease));
}
/** Raw field-105 ambient-light records, delivered on the main queue. */
addAmbientLightListener(listener: (body: Uint8Array) => void): () => void {
const proxy = AmbientLightListener.new() as AmbientLightListener;
proxy.handler = listener;
const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy);
return () => { this.communicator.removeAmbientLightListenerListener(handle); };
}
/** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */
addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void {
const proxy = MicStatusListener.new() as MicStatusListener;
proxy.handler = listener;
const handle = this.communicator.addMicStatusListenerNsDataListener(proxy);
return () => { this.communicator.removeMicStatusListenerListener(handle); };
}
/** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */
sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean {
return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple(
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
}
/**
* Every audio-characteristic packet, whatever enabled the mics, on the
* main queue. Passive: sends nothing and doesn't affect capture.
*/
addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void {
const proxy = AudioPacketListener.new() as AudioPacketListener;
proxy.handler = listener;
const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy);
return () => { this.communicator.removeAudioMonitorListenerListener(handle); };
}
/** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */
isAudioCaptureActive(): boolean {
return Boolean(this.communicator.isAudioCaptureActive());
}
/**
* Serialized glasses microphone enable/disable (the voice bridge's
* IosMicrophoneSession contract). A release while an enable awaits its ACK
+17
View File
@@ -0,0 +1,17 @@
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
export type ImuReading = { x: number; y: number; z: number; source: number };
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
case 1:
return "R arm";
case 2:
return "ring";
case 3:
return "L arm";
default:
return "device";
}
}
+34
View File
@@ -0,0 +1,34 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
import { type ImuReading } from "./imu-types";
export * from "./imu-types";
/**
* IMU (accelerometer) access on iOS: the same API as imu.ts over the running
* shared-Kotlin session. As there, the stream is experimental, the calls are
* no-ops while no session is running, and a listener stays with the session
* it subscribed to.
*/
/** Enable/disable the accelerometer report stream. No-op if not connected. */
export function setImuReportEnabled(enable: boolean, reportFreq: number): void {
try {
FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq);
} catch (error) {
console.warn(`setImuReportEnabled failed: ${error}`);
}
}
/** Subscribe to IMU readings; returns an unsubscribe function. */
export function addImuListener(listener: (reading: ImuReading) => void): () => void {
const active = FaceclawCommunicatorBridge.getActive();
if (!active) return () => {};
const unsubscribe = active.addImuListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may be gone after a disconnect; nothing to clean up.
}
};
}
+3 -15
View File
@@ -12,7 +12,9 @@ declare const com: any;
* static, so it works from any isolate without threading the bridge through.
*/
export type ImuReading = { x: number; y: number; z: number; source: number };
import { type ImuReading } from "./imu-types";
export * from "./imu-types";
function activeCommunicator(): any {
if (!global.isAndroid) return null;
@@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v
}
};
}
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
case 1:
return "R arm";
case 2:
return "ring";
case 3:
return "L arm";
default:
return "device";
}
}
+30
View File
@@ -0,0 +1,30 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/** The iOS twin of mic-control-link.ts, over the running shared-Kotlin session. */
export function micControlSession(): unknown {
return FaceclawCommunicatorBridge.getActive();
}
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
export function sendMicControlRecord(
session: unknown,
record: Uint8Array,
label: string,
rightTemple: boolean,
leftTemple: boolean,
): boolean {
return (session as FaceclawCommunicatorBridge).sendFaceclawMicControl(record, label, rightTemple, leftTemple);
}
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
const unsubscribe = (session as FaceclawCommunicatorBridge).addMicStatusListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a reconnect.
}
};
}
+46
View File
@@ -0,0 +1,46 @@
import { toJavaBytes } from "./cloud-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
/**
* Platform plumbing for the CFW mic_control channel (apps/microphones/mic-control.ts):
* the running session to send records on and to hear field-104 status replies
* from. The session is opaque and is a different object after a reconnect.
*/
export function micControlSession(): unknown {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
export function sendMicControlRecord(
session: unknown,
record: Uint8Array,
label: string,
rightTemple: boolean,
leftTemple: boolean,
): boolean {
return (session as any).sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
}
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
const communicator = session as any;
const proxy = new com.faceclaw.app.FaceclawMicStatusListener({
onMicStatus: (body: any, arm: string) => listener(toUint8Array(body), String(arm)),
});
communicator.addMicStatusListener(proxy);
return () => {
try {
communicator.removeMicStatusListener(proxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
};
}
@@ -24,6 +24,16 @@ interface IosAudioPacketListener {
fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long)
}
/** CFW ambient-light reports (the raw field-105 record) as NSData. Main queue. Named apart like [IosAudioPacketListener]. */
interface IosAmbientLightListener {
fun onNsDataAmbientLight(body: NSData)
}
/** CFW mic_control status records (the raw field-104 record) as NSData, with the temple: "L", "R" or "?". Main queue. */
interface IosMicStatusListener {
fun onNsDataMicStatus(body: NSData, arm: String)
}
/**
* ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS
* counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes
@@ -188,6 +198,39 @@ class IosGlassesSession internal constructor(
fun sendFaceclawMicControl(record: NSData?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
core.sendFaceclawMicControl(record?.byteArray(), label, rightTemple, leftTemple)
// NSData twins of the ByteArray listeners above (a Kotlin ByteArray crosses into JavaScript one
// byte per call). The core already delivers these on the main queue. Each returns the adapter it
// registered: pass that to the matching remove* method to unsubscribe.
fun addAmbientLightListenerNsData(listener: IosAmbientLightListener): FaceclawAmbientLightListener {
val adapter = object : FaceclawAmbientLightListener {
override fun onAmbientLight(body: ByteArray?) = listener.onNsDataAmbientLight((body ?: ByteArray(0)).data())
}
core.addAmbientLightListener(adapter)
return adapter
}
fun addMicStatusListenerNsData(listener: IosMicStatusListener): FaceclawMicStatusListener {
val adapter = object : FaceclawMicStatusListener {
override fun onMicStatus(body: ByteArray?, arm: String?) =
listener.onNsDataMicStatus((body ?: ByteArray(0)).data(), arm ?: "?")
}
core.addMicStatusListener(adapter)
return adapter
}
/** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */
fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener {
val adapter = object : FaceclawAudioPacketListener {
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) =
listener.onNsDataAudioPacket((data ?: ByteArray(0)).data(), arm ?: "?", arrivalMs)
}
core.addAudioMonitorListener(adapter)
return adapter
}
fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.removeAudioMonitorListener(listener)
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean =
core.startG2AudioForwarding(listener?.let { MainQueueAudioListener(it) })
+2 -1
View File
@@ -32,7 +32,8 @@ function harness(refused = new Set()) {
NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; },
FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) },
FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {},
FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} },
FaceclawKitFaceclawCompassListener: {}, FaceclawKitFaceclawImuListener: {}, FaceclawKitIosAmbientLightListener: {},
FaceclawKitIosMicStatusListener: {}, console: { log() {}, warn() {} },
}, {
'@nativescript/core': { File: {}, knownFolders: {}, path: {} },
'../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 },
+186
View File
@@ -0,0 +1,186 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
// The developer demos' sensor plumbing on iOS: FaceclawCommunicatorBridge over a
// fake IosGlassesSession, and the imu/ambient-light/audio-monitor/mic-control-link
// .ios.ts modules over that bridge.
function harness() {
const calls = [];
const listeners = { imu: new Set(), light: new Set(), status: new Set(), audio: new Set() };
// Like the Kotlin facade, the NSData add* methods return an adapter handle for remove*.
const handleFor = (set, proxy) => { const handle = { proxy }; set.add(handle); return handle; };
const session = {
setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {},
start() { calls.push(['start']); return true; },
close() { calls.push(['close']); },
setImuReportEnabledEnableReportFrq(enable, freq) { calls.push(['imu', enable, freq]); },
addImuListenerListener(proxy) { listeners.imu.add(proxy); },
removeImuListenerListener(proxy) { listeners.imu.delete(proxy); },
queryAmbientLight() { calls.push(['als-query']); },
setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(...args) { calls.push(['als-poll', ...args]); },
addAmbientLightListenerNsDataListener: proxy => handleFor(listeners.light, proxy),
removeAmbientLightListenerListener(handle) { listeners.light.delete(handle); },
addMicStatusListenerNsDataListener: proxy => handleFor(listeners.status, proxy),
removeMicStatusListenerListener(handle) { listeners.status.delete(handle); },
addAudioMonitorListenerNsDataListener: proxy => handleFor(listeners.audio, proxy),
removeAudioMonitorListenerListener(handle) { listeners.audio.delete(handle); },
isAudioCaptureActive: () => true,
sendFaceclawMicControlRecordLabelRightTempleLeftTemple(record, label, right, left) {
calls.push(['mic-control', Array.from(record), label, right, left]);
return label !== 'refused';
},
};
class NSObject { static new() { return new this(); } }
const load = loader({
NSObject, NativeClass: () => cls => cls, setTimeout, clearTimeout,
FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) },
FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {},
FaceclawKitFaceclawCompassListener: {}, FaceclawKitFaceclawImuListener: {}, FaceclawKitIosAmbientLightListener: {},
FaceclawKitIosMicStatusListener: {}, console: { log() {}, warn() {} },
}, {
'@nativescript/core': { File: {}, knownFolders: {}, path: {} },
'../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 },
// NSData stands in as the bytes themselves.
'./kotlin-data': { toData: bytes => bytes, fromData: data => new Uint8Array(data) },
'./ios-graphics': { previewPixels() {} },
'./ios-bluetooth': { iosBluetooth: {} },
});
const bridgeModule = load('app/native/faceclaw-communicator.ios.ts');
return { load, session, calls, listeners, FaceclawCommunicatorBridge: bridgeModule.FaceclawCommunicatorBridge };
}
function alsRecord(alsValue) {
const body = new Uint8Array(24);
body.set([0x41, 0x4c, 1, 2, 0x0b]);
body[6] = alsValue & 0xff; body[7] = alsValue >> 8;
return body;
}
test('the bridge is the active session from start() until close()', async () => {
const h = harness();
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
assert.equal(h.FaceclawCommunicatorBridge.getActive(), null, 'not before start');
await bridge.start();
assert.equal(h.FaceclawCommunicatorBridge.getActive(), bridge);
const closing = bridge.close();
assert.equal(h.FaceclawCommunicatorBridge.getActive(), null, 'cleared as soon as close() is called');
await closing;
assert.deepEqual(h.calls, [['start'], ['close']]);
});
test('a start queued behind close() does not make a closed bridge active', async () => {
const h = harness();
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
const started = bridge.start();
const closed = bridge.close();
await Promise.all([started, closed]);
assert.equal(h.FaceclawCommunicatorBridge.getActive(), null);
});
test('the sensor modules are no-ops without a running session', () => {
const h = harness();
const imu = h.load('app/native/imu.ios.ts');
const light = h.load('app/native/ambient-light.ios.ts');
const monitor = h.load('app/native/audio-monitor.ios.ts');
imu.setImuReportEnabled(true, 200);
light.setAmbientLightPolling(true);
light.queryAmbientLight();
imu.addImuListener(() => assert.fail('no session'))();
light.addAmbientLightListener(() => assert.fail('no session'))();
assert.equal(monitor.audioMonitorCommunicator(), null);
assert.equal(monitor.isFaceclawAudioCaptureActive(), false);
monitor.addAudioMonitorListeners(null, () => {}, () => {})();
assert.deepEqual(h.calls, []);
});
test('IMU control and samples go through the active session', async () => {
const h = harness();
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
await bridge.start();
const imu = h.load('app/native/imu.ios.ts');
imu.setImuReportEnabled(true, 199.6);
imu.setImuReportEnabled(false, -5);
const readings = [];
const off = imu.addImuListener(reading => readings.push({ ...reading }));
const [proxy] = h.listeners.imu;
proxy.onImuDataXYZEventSource(0.5, -1, 9.75, 3);
off();
assert.equal(h.listeners.imu.size, 0);
assert.deepEqual(readings, [{ x: 0.5, y: -1, z: 9.75, source: 3 }]);
assert.equal(imu.imuSourceLabel(3), 'L arm');
assert.deepEqual(h.calls.slice(1), [['imu', true, 200], ['imu', false, 0]]);
});
test('ambient light: defaults, query, and decoded reports over NSData', async () => {
const h = harness();
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
await bridge.start();
const light = h.load('app/native/ambient-light.ios.ts');
light.setAmbientLightPolling(true, { intervalMs: 250, minDelta: 0, heartbeatMs: 1000, bindToLease: true });
light.setAmbientLightPolling(false);
light.queryAmbientLight();
assert.deepEqual(h.calls.slice(1), [
['als-poll', true, 250, 0, 1000, true],
['als-poll', false, 500, 1, 5000, true],
['als-query'],
]);
const reports = [];
const off = light.addAmbientLightListener(report => reports.push(report));
const [handle] = h.listeners.light;
handle.proxy.onNsDataAmbientLightBody(alsRecord(1234));
handle.proxy.onNsDataAmbientLightBody(new Uint8Array([1, 2, 3])); // not a report: dropped
off();
assert.equal(h.listeners.light.size, 0, 'removed by the handle the session returned');
assert.equal(reports.length, 1);
assert.equal(reports[0].alsValue, 1234);
assert.equal(reports[0].reason, 'poll');
assert.equal(reports[0].opened, true);
assert.equal(reports[0].passive, true);
assert.equal(reports[0].readOk, true);
});
test('the passive audio monitor sees packets and mic status per temple', async () => {
const h = harness();
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
await bridge.start();
const monitor = h.load('app/native/audio-monitor.ios.ts');
const communicator = monitor.audioMonitorCommunicator();
assert.equal(communicator, bridge);
assert.equal(monitor.isFaceclawAudioCaptureActive(), true);
const packets = [], statuses = [];
const off = monitor.addAudioMonitorListeners(communicator,
(data, arm, arrivalMs) => packets.push([Array.from(data), arm, arrivalMs]),
(body, arm) => statuses.push([Array.from(body), arm]));
const [audio] = h.listeners.audio, [status] = h.listeners.status;
audio.proxy.onNsDataAudioPacketDataArmArrivalMs(new Uint8Array([1, 2]), 'L', 42);
audio.proxy.onNsDataAudioPacketDataArmArrivalMs(new Uint8Array([3]), '', 43);
status.proxy.onNsDataMicStatusBodyArm(new Uint8Array([0x4d, 0x43]), 'R');
off();
assert.equal(h.listeners.audio.size + h.listeners.status.size, 0);
assert.deepEqual(packets, [[[1, 2], 'L', 42], [[3], '?', 43]]);
assert.deepEqual(statuses, [[[0x4d, 0x43], 'R']]);
assert.deepEqual(h.calls, [['start']], 'monitoring sends nothing');
});
test('mic-control link sends records and hears status on iOS', async () => {
const h = harness();
const link = h.load('app/native/mic-control-link.ios.ts');
assert.equal(link.micControlSession(), null);
const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
await bridge.start();
const session = link.micControlSession();
assert.equal(session, bridge);
assert.equal(link.sendMicControlRecord(session, new Uint8Array([0x4d, 0x43, 1, 4]), 'query', true, true), true);
assert.equal(link.sendMicControlRecord(session, new Uint8Array([0x4d, 0x43, 1, 4]), 'refused', true, false), false);
const heard = [];
const off = link.addMicStatusListener(session, (body, arm) => heard.push([Array.from(body), arm]));
[...h.listeners.status][0].proxy.onNsDataMicStatusBodyArm(new Uint8Array([7]), 'L');
off();
assert.equal(h.listeners.status.size, 0);
assert.deepEqual(heard, [[[7], 'L']]);
assert.deepEqual(h.calls.slice(1), [
['mic-control', [0x4d, 0x43, 1, 4], 'query', true, true],
['mic-control', [0x4d, 0x43, 1, 4], 'refused', true, false],
]);
});
@@ -197,6 +197,52 @@ class IosSessionFacadeTest {
assertFalse(pumpMainUntil(200) { relayed.size > 1 })
}
@Test
fun nsDataSensorListenersDeliverOnMainAndUnsubscribeByTheirHandles() {
val session = IosGlassesSession(FakeIosLink(), null, IosSessionHost(threadName = "test-session-sensors"), RIGHT, LEFT, "", IosProtocolPlatform)
val core = session.coreForTests()
val light = ArrayList<List<Byte>>()
val status = ArrayList<Pair<List<Byte>, String>>()
val audio = ArrayList<Triple<List<Byte>, String, Long>>()
var offMain = 0
val lightHandle = session.addAmbientLightListenerNsData(object : IosAmbientLightListener {
override fun onNsDataAmbientLight(body: platform.Foundation.NSData) {
if (!NSThread.isMainThread) offMain++
light.add(body.byteArray().toList())
}
})
val statusHandle = session.addMicStatusListenerNsData(object : IosMicStatusListener {
override fun onNsDataMicStatus(body: platform.Foundation.NSData, arm: String) {
if (!NSThread.isMainThread) offMain++
status.add(body.byteArray().toList() to arm)
}
})
val audioHandle = session.addAudioMonitorListenerNsData(object : IosAudioPacketListener {
override fun onNsDataAudioPacket(data: platform.Foundation.NSData, arm: String, arrivalMs: Long) {
if (!NSThread.isMainThread) offMain++
audio.add(Triple(data.byteArray().toList(), arm, arrivalMs))
}
})
val report = byteArrayOf(0x41, 0x4c, 1, 2)
val record = byteArrayOf(0x4d, 0x43, 1)
val packet = byteArrayOf(9, 8, 7)
core.emitAmbientLight(report)
core.emitMicStatus(record, LEFT)
core.emitAudioMonitorPacket(packet, "R", 1234L)
assertTrue(pumpMainUntil(2_000) { light.size == 1 && status.size == 1 && audio.size == 1 })
assertEquals(report.toList(), light[0])
assertEquals(record.toList() to "L", status[0])
assertEquals(Triple(packet.toList(), "R", 1234L), audio[0])
assertEquals(0, offMain)
session.removeAmbientLightListener(lightHandle)
session.removeMicStatusListener(statusHandle)
session.removeAudioMonitorListener(audioHandle)
core.emitAmbientLight(report)
core.emitMicStatus(record, RIGHT)
core.emitAudioMonitorPacket(packet, "L", 5678L)
assertFalse(pumpMainUntil(200) { light.size > 1 || status.size > 1 || audio.size > 1 })
}
@Test
fun hostDispatchesToTheCallingQueueAndRunsTheWorker() {
val host = IosSessionHost(threadName = "test-host")