Merge branch 'main' into ble-bandwidth-test

This commit is contained in:
James Babcock committed 2026-09-09 21:59:38 -07:00
commit 50cbc3a756
198 files changed
+15988 -2899

No files matched your search

+154
View File
@@ -0,0 +1,154 @@
import { Utils } from "@nativescript/core";
declare const com: any;
/**
* Bridge to the Java alarm clock (FaceclawAlarms / FaceclawAlarmService):
* the durable phone side of the Timers app's timers and alarms. The phone
* schedules an exact alarm-clock alarm per item, re-arms them after reboots
* and time changes, and when one comes due shows it on the phone (silently
* at first) and adds phone sound if the glasses cannot carry it. Ids are the
* engine's item ids.
*/
export type AlarmKind = "timer" | "alarm";
export type PhoneAlarmAction = { id: number; action: "dismiss" | "snooze"; minutes: number; atMs: number };
export type AlarmReliabilityIssue = { code: string; message: string; fixable: boolean };
function getContext(): android.content.Context | null {
if (!global.isAndroid) return null;
return Utils.android.getApplicationContext() ?? null;
}
/** Schedule (or move) the phone alarm that rings `title` / `text` at triggerAtMs. */
export function schedulePhoneAlarm(
itemId: number,
triggerAtMs: number,
title: string,
text: string,
kind: AlarmKind,
snoozeMinutes: number,
): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.schedule(context, itemId, triggerAtMs, title, text, kind, snoozeMinutes);
}
/** Cancel the pending phone alarm, stop it if ringing, and clear its notification. */
export function cancelPhoneAlarm(itemId: number): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.cancel(context, itemId);
}
/** The engine's own expiry path: start the phone ringing flow (deduplicated against the alarm's). */
export function ringPhoneAlarm(itemId: number, title: string, text: string, kind: AlarmKind, snoozeMinutes: number): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.ring(context, itemId, title, text, kind, snoozeMinutes);
}
/** The glasses are showing the item: the phone starts its 30-second acknowledgement clock. */
export function phoneAlarmDeliveredToGlasses(itemId: number): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.deliveredToGlasses(context, itemId);
}
/** The wearer dealt with the item on the glasses: the phone goes quiet. */
export function acknowledgePhoneAlarm(itemId: number): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.acknowledge(context, itemId);
}
/** Tell the phone whether the glasses could carry an alarm right now. */
export function setPhoneAlarmGlassesStatus(connected: boolean, worn: boolean, charging: boolean): void {
if (!global.isAndroid) return;
com.faceclaw.app.FaceclawAlarms.setGlassesStatus(connected, worn, charging);
}
let retainedListenerProxy: any = null;
const actionListeners = new Set<(action: PhoneAlarmAction) => void>();
/** Live phone-side dismiss / snooze events (journaled ones arrive through drainPhoneAlarmJournal). */
export function onPhoneAlarmAction(listener: (action: PhoneAlarmAction) => void): () => void {
if (retainedListenerProxy === null && global.isAndroid) {
retainedListenerProxy = new com.faceclaw.app.FaceclawAlarmListener({
onPhoneAction: (id: number, action: string, minutes: number) => {
const event: PhoneAlarmAction = {
id: Number(id),
action: String(action) === "snooze" ? "snooze" : "dismiss",
minutes: Number(minutes),
atMs: Date.now(),
};
for (const registered of Array.from(actionListeners)) {
try {
registered(event);
} catch (error) {
console.warn(`phone alarm action listener failed: ${error}`);
}
}
},
});
com.faceclaw.app.FaceclawAlarms.setListener(retainedListenerProxy);
}
actionListeners.add(listener);
return () => {
actionListeners.delete(listener);
};
}
/** Phone-side actions taken while the engine was not listening, oldest first; cleared on read. */
export function drainPhoneAlarmJournal(): PhoneAlarmAction[] {
const context = getContext();
if (!context) return [];
let parsed: any;
try {
parsed = JSON.parse(String(com.faceclaw.app.FaceclawAlarms.drainJournal(context)));
} catch {
return [];
}
if (!Array.isArray(parsed)) return [];
return parsed
.filter((entry) => entry && typeof entry === "object" && Number.isFinite(Number(entry.id)))
.map((entry) => ({
id: Number(entry.id),
action: entry.action === "snooze" ? "snooze" : "dismiss",
minutes: Number(entry.minutes) || 0,
atMs: Number(entry.at) || 0,
}));
}
/** Conditions under which the phone may fail to ring, most serious first. */
export function checkPhoneAlarmReliability(): AlarmReliabilityIssue[] {
const context = getContext();
if (!context) return [];
let parsed: any;
try {
parsed = JSON.parse(String(com.faceclaw.app.FaceclawAlarms.checkReliability(context)));
} catch (error) {
console.warn(`alarm reliability check failed: ${error}`);
return [];
}
if (!Array.isArray(parsed)) return [];
return parsed
.filter((entry) => entry && typeof entry === "object" && typeof entry.code === "string")
.map((entry) => ({ code: entry.code, message: String(entry.message ?? ""), fixable: entry.fixable !== false }));
}
/** Open the system screen where a reliability issue can be fixed. */
export function openPhoneAlarmReliabilityFix(code: string): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawAlarms.openReliabilityFix(context, code);
}
/** The phone's ring / miss / action log, one line per entry, newest last. */
export function readPhoneAlarmLog(): string {
const context = getContext();
if (!context) return "";
return String(com.faceclaw.app.FaceclawAlarms.readLog(context));
}
+157
View File
@@ -0,0 +1,157 @@
/**
* Ambient light sensor access through the custom firmware (CFW image-handler
* mode 16, capability token "als16"; see g2flash/patches/als_sensor.c).
*
* The stock firmware only reads its OPT3001 light sensor while auto-brightness
* is on, and then steps the panel brightness itself (visible flicker). The CFW
* adds two things:
*
* - QUERY: one report from the driver's current state.
* - PASSIVE polling: the firmware opens and polls the sensor at our interval,
* the stock adjuster never touches the panel, and a report arrives when the
* reading moved by at least `minDelta` or `heartbeatMs` elapsed. The phone
* then applies brightness itself (setBrightness with autoAdjust=false), e.g.
* only while the display is blank or during a UI transition.
*
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
*/
import { toUint8Array } from "../util/array-util";
declare const com: any;
declare const global: any;
export const ALS_REPORT_VERSION = 1;
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
export type AmbientLightReport = {
reason: AmbientLightReason;
/** The sensor driver is open (readings are live). */
opened: boolean;
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
passive: boolean;
/** The user's stock auto-brightness setting is on. */
autoBrightness: boolean;
/** The last passive read succeeded (only meaningful while passive). */
readOk: boolean;
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
status: number;
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
alsValue: number;
/** Max of the last five readings (what the stock curve is applied to). */
peak: number;
/** Brightness level (0..100) the stock curve would choose for `peak`. */
stockTarget: number;
/** Current brightness setting (0..100). */
brightness: number;
/** Row of the stock brightness curve (0..5) for `peak`. */
gear: number;
/** Stock learned scale factor, Q10 (1024 = 1.0). */
scaleQ10: number;
/** Firmware millisecond tick when the report was built. */
tickMs: number;
};
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
function rd16(b: Uint8Array, i: number): number {
return b[i] | (b[i + 1] << 8);
}
function rd32(b: Uint8Array, i: number): number {
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
}
/** Decode a field-105 record body, or null when it isn't one. */
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
return null;
}
const flags = body[4];
return {
reason: REASONS[body[3]] ?? "poll",
opened: (flags & 0x01) !== 0,
passive: (flags & 0x02) !== 0,
autoBrightness: (flags & 0x04) !== 0,
readOk: (flags & 0x08) !== 0,
status: body[5],
alsValue: rd32(body, 6),
peak: rd32(body, 10),
stockTarget: rd16(body, 14),
brightness: body[16],
gear: body[17],
scaleQ10: rd16(body, 18),
tickMs: rd32(body, 20),
};
}
export type AmbientLightPollingOptions = {
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
intervalMs?: number;
/** Report when the reading moved by at least this much. Default 1. */
minDelta?: number;
/** Also report after this many ms without one (0 = never). Default 5000. */
heartbeatMs?: number;
/**
* Stop automatically when the Faceclaw framebuffer lease is released or
* lapses (the phone went away). Default true.
*/
bindToLease?: boolean;
};
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Ask the firmware for one ambient-light report (delivered to listeners). */
export function queryAmbientLight(): void {
try {
activeCommunicator()?.queryAmbientLight();
} catch (error) {
console.warn(`queryAmbientLight failed: ${error}`);
}
}
/**
* Start or stop CFW passive light-sensor polling. Starting is idempotent and
* re-opens the sensor if the stock firmware closed it, so it is safe to resend
* on every session start.
*/
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
try {
activeCommunicator()?.setAmbientLightPolling(
enabled,
options.intervalMs ?? 500,
options.minDelta ?? 1,
options.heartbeatMs ?? 5000,
options.bindToLease ?? true,
);
} catch (error) {
console.warn(`setAmbientLightPolling failed: ${error}`);
}
}
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
const active = activeCommunicator();
if (!active) return () => {};
const proxy = new com.faceclaw.app.FaceclawAmbientLightListener({
onAmbientLight: (body: any) => {
const report = decodeAmbientLightReport(toUint8Array(body));
if (report) listener(report);
},
});
active.addAmbientLightListener(proxy);
return () => {
try {
activeCommunicator()?.removeAmbientLightListener(proxy);
} catch {
// The communicator may have been replaced during a disconnect.
}
};
}
+16
View File
@@ -0,0 +1,16 @@
declare const com: any;
export interface BleTrafficSample {
/** Outbound messages (writeFrames calls) since process start. */
messages: number;
/** Outbound payload bytes (sum of frame sizes) since process start. */
bytes: number;
/** Display frames whose last message was acked, since process start. */
frames: number;
}
/** Running totals of outbound BLE traffic, counted process-wide in FaceclawBleManager. */
export function sampleBleTraffic(): BleTrafficSample {
const raw = com.faceclaw.app.FaceclawBleManager.sampleOutboundTraffic();
return { messages: Number(raw[0]), bytes: Number(raw[1]), frames: Number(raw[2]) };
}
+7
View File
@@ -9,6 +9,9 @@
export type CloudSttTranscriptEvent = {
text: string;
isFinal: boolean;
/** Optional presentation for Transcribe only; dictation uses plain text. */
transcribeText?: string;
paragraphBreakAfter?: boolean;
};
export type CloudSttOptions = {
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
onReady?: () => void;
onDisconnected?: (message: string) => void;
};
export interface CloudSttClient {
@@ -25,6 +30,8 @@ export interface CloudSttClient {
acceptPcm(pcm: Uint8Array): void;
/** End of utterance: produce a final transcript. */
finish(): void;
/** Commit a pause boundary while keeping the audio session open. */
commitSegment?(): void;
/** Abandon the session without finalizing. */
stop(): void;
}
+34 -6
View File
@@ -5,10 +5,23 @@ export const COMPASS_CHANGED = 15;
export const COMPASS_CALIBRATION_STARTED = 16;
export const COMPASS_CALIBRATION_COMPLETE = 17;
export type CompassDiagnostics = {
/** 0 uncalibrated .. 3 well calibrated; -1 unavailable. */
magneticAccuracy: number;
/** 0 none, 1 small/temperature, 2 large; -1 unavailable. */
magneticAnomalies: number;
/** 0 unknown, 1 GRV (relative), 2 GMRV, 3 RV. */
orientationSource: number;
flags: number;
sampleTimeMs: number;
};
export type CompassEvent = {
command: number;
/** Magnetic heading in degrees, or -1 for calibration-only events. */
headingDegrees: number;
/** Absent with stock/older CFW or an unrecognized diagnostic extension. */
diagnostics?: CompassDiagnostics;
};
function activeCommunicator(): any {
@@ -20,22 +33,37 @@ function activeCommunicator(): any {
}
}
/** Request CFW mode 10 to start/stop the stock firmware compass. */
export function setCompassEnabled(enabled: boolean): void {
/**
* Request CFW mode 10 to start/stop the stock firmware compass on behalf of
* `owner`. The magnetometer is shared: it runs while any owner wants it and
* stops once the last one releases it, so apps can't switch each other off.
*/
export function setCompassEnabled(enabled: boolean, owner = "compass"): void {
try {
activeCommunicator()?.setCompassEnabled(enabled);
activeCommunicator()?.setCompassEnabled(owner, enabled);
} catch (error) {
console.warn(`setCompassEnabled failed: ${error}`);
}
}
/** Subscribe to stock compass heading/calibration notifications. */
/**
* Subscribe to stock compass heading/calibration notifications. Events are
* delivered on the calling thread's Looper, so this works from app workers.
*/
export function addCompassListener(listener: (event: CompassEvent) => void): () => void {
const active = activeCommunicator();
if (!active) return () => {};
const proxy = new com.faceclaw.app.FaceclawCompassListener({
onCompassEvent: (command: number, headingDegrees: number) => {
listener({ command: Number(command), headingDegrees: Number(headingDegrees) });
onCompassEvent: (command: number, headingDegrees: number, magneticAccuracy: number,
magneticAnomalies: number, orientationSource: number, diagnosticFlags: number, sampleTimeMs: number) => {
listener({
command: Number(command), headingDegrees: Number(headingDegrees),
diagnostics: diagnosticFlags >= 0 ? {
magneticAccuracy: Number(magneticAccuracy), magneticAnomalies: Number(magneticAnomalies),
orientationSource: Number(orientationSource), flags: Number(diagnosticFlags),
sampleTimeMs: Number(sampleTimeMs),
} : undefined,
});
},
});
active.addCompassListener(proxy);
+45 -44
View File
@@ -1,4 +1,4 @@
import { encodeBase64 } from "./cloud-stt";
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
declare const com: any;
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
const MODEL_ID = "scribe_v2_realtime";
const SAMPLE_RATE = 16000;
export type ElevenLabsTranscriptEvent = {
text: string;
isFinal: boolean;
};
export type ElevenLabsSttOptions = {
apiKey: string;
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
};
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
export type ElevenLabsSttOptions = CloudSttOptions;
export class ElevenLabsSttClient {
private ws: any = null;
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
// PCM that arrived before the socket finished opening; flushed on open.
private readonly pendingChunks: string[] = [];
private latestText = "";
private readonly commits: boolean[] = [];
constructor(private readonly options: ElevenLabsSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
},
});
try {
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
this.options.onStatus("Connecting to ElevenLabs...");
} catch (error) {
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: flush and commit for a final transcript. */
finish(): void {
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
const commitMessage = JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: "",
commit: true,
sample_rate: SAMPLE_RATE,
});
if (this.open) {
this.trySend(commitMessage);
} else {
this.pendingChunks.push("__commit__");
}
this.commits.push(paragraphBreakAfter);
if (this.open) this.sendChunk("__commit__");
else this.pendingChunks.push("__commit__");
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.finish();
return;
}
this.trySend(
JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: base64OrCommit,
commit: false,
sample_rate: SAMPLE_RATE,
}),
);
const commit = base64OrCommit === "__commit__";
this.trySend(JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: commit ? "" : base64OrCommit,
commit,
sample_rate: SAMPLE_RATE,
}));
}
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("ElevenLabs send failed.");
}
} catch (error) {
console.warn("elevenlabs send failed", error);
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
}
}
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
this.options.onTranscript({ text: this.latestText, isFinal: false });
return;
case "committed_transcript":
case "committed_transcript_with_timestamps":
this.latestText = String(message.text ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
this.latestText = "";
return;
case "committed_transcript_with_timestamps":
// The timestamp event follows the plain commit; do not append twice.
return;
case "rate_limited":
this.options.onDisconnected?.("ElevenLabs rate limited.");
return;
case "error":
case "auth_error":
case "quota_exceeded":
case "rate_limited":
case "input_error":
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
return;
+22 -1
View File
@@ -22,6 +22,8 @@ export type CommunicatorState = {
export type HeadsetBatteryState = {
battery: number;
chargingStatus: number;
ringBattery?: number;
ringChargingStatus?: number;
};
export type FrameMetrics = {
@@ -120,6 +122,11 @@ function nonNegativeNumber(value: number): number {
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
}
/** A 0..1 brightness factor as the compositor's 0..256 fixed-point form. */
export function dimFactor256(factor: number): number {
return Math.round(Math.max(0, Math.min(1, factor)) * 256);
}
export class FaceclawCommunicatorBridge {
private readonly communicator: any;
private readonly listenerProxy: any;
@@ -184,10 +191,12 @@ export class FaceclawCommunicatorBridge {
frameTimings.logFrame(event.frameId, "input event received on JS side");
this.emitAsync(this.ringListeners, event);
},
onBatteryState: (headsetBattery: number, headsetCharging: number) => {
onBatteryState: (headsetBattery: number, headsetCharging: number, ringBattery: number, ringCharging: number) => {
const state = {
battery: Number(headsetBattery),
chargingStatus: Number(headsetCharging),
ringBattery: Number(ringBattery),
ringChargingStatus: Number(ringCharging),
};
this.emitAsync(this.batteryListeners, state);
},
@@ -524,6 +533,18 @@ export class FaceclawCommunicatorBridge {
});
}
/**
* Dim every compositor surface whose zOrder is below `belowZOrder` to
* `factor` (0..1; 1 = no dimming) of its brightness; takes effect at the
* next composite. How a shell overlay's Layer.dimUnderneath reaches the
* window surfaces beneath the shell surface.
*/
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
await this.enqueueJavaCall(() => {
this.communicator.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
});
}
/** Show or hide a compositor surface; takes effect at the next composite. */
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueJavaCall(() => {
+33 -4
View File
@@ -6,20 +6,32 @@ export type FlashPromptState =
| "connecting"
| "connected"
| "prompting"
| "battery"
| "result"
| "cancelled"
| "timeout"
| "disconnected"
| "error";
/** Per-arm battery percent; null when that arm didn't answer. */
export type FlashPromptBattery = { right: number | null; left: number | null };
function normalizePercent(value: number): number | null {
const n = Number(value);
return Number.isFinite(n) && n >= 0 ? n : null;
}
/**
* TS wrapper around the native FaceclawFlashPromptCommunicator — the
* stock-firmware-compatible BLE path that shows the pre-flash confirmation on
* the glasses and reports the user's Yes/No choice. Separate from
* FaceclawCommunicatorBridge on purpose (different, minimal protocol subset).
*
* Talks to the right arm only: the lenses relay messages between themselves,
* and acks/events always come from the right arm.
* Connects and authenticates both arms (so any OS pairing prompts happen up
* front, before flashing), shows the prompt via the right arm (the lenses relay
* messages between themselves, and acks/events always come from the right
* arm), then reads each arm's battery. With `skipPrompt` the on-glasses
* confirmation is omitted and only the battery is read.
*/
export class FlashPromptCommunicator {
private readonly communicator: any;
@@ -27,20 +39,32 @@ export class FlashPromptCommunicator {
private readonly logListeners = new Set<(line: string) => void>();
private readonly stateListeners = new Set<(state: FlashPromptState, detail: string) => void>();
private readonly resultListeners = new Set<(approved: boolean) => void>();
private readonly batteryListeners = new Set<(battery: FlashPromptBattery) => void>();
constructor(rightAddress: string, warningText: string) {
constructor(
addresses: { right: string; left: string },
warningText: string,
options?: { skipPrompt?: boolean },
) {
const context = Utils.android.getApplicationContext();
if (!context) throw new Error("Android application context unavailable");
this.communicator = new com.faceclaw.app.FaceclawFlashPromptCommunicator(
context,
rightAddress,
addresses.right,
addresses.left,
warningText,
Boolean(options?.skipPrompt),
);
this.listenerProxy = new com.faceclaw.app.FaceclawFlashPromptListener({
onLog: (line: string) => this.emit(this.logListeners, String(line)),
onState: (state: string, detail: string) =>
this.emit(this.stateListeners, String(state) as FlashPromptState, String(detail ?? "")),
onBattery: (rightPercent: number, leftPercent: number) =>
this.emit(this.batteryListeners, {
right: normalizePercent(rightPercent),
left: normalizePercent(leftPercent),
}),
onResult: (approved: boolean) => this.emit(this.resultListeners, Boolean(approved)),
});
this.communicator.setListener(this.listenerProxy);
@@ -56,6 +80,11 @@ export class FlashPromptCommunicator {
return () => this.stateListeners.delete(listener);
}
onBattery(listener: (battery: FlashPromptBattery) => void): () => void {
this.batteryListeners.add(listener);
return () => this.batteryListeners.delete(listener);
}
onResult(listener: (approved: boolean) => void): () => void {
this.resultListeners.add(listener);
return () => this.resultListeners.delete(listener);
+13 -4
View File
@@ -84,6 +84,7 @@ type TerminalDataKind = "snapshot" | "output";
export class G2MirrorClient {
private ws: any = null;
private listenerProxy: any = null;
private connectionGeneration = 0;
private phase: G2MirrorPhase = "idle";
private status = "Not connected.";
private sessions: G2MirrorSession[] = [];
@@ -168,14 +169,21 @@ export class G2MirrorClient {
return () => this.titleListeners.delete(listener);
}
/** Grid dimensions for the next connection's handshake. */
setViewport(cols: number, rows: number): void {
this.options.cols = cols;
this.options.rows = rows;
}
start(): void {
if (this.ws) return;
this.stopped = false;
const generation = ++this.connectionGeneration;
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.stopped) return;
if (this.stopped || generation !== this.connectionGeneration) return;
this.setState("connecting", "Authenticating...");
this.send({
type: "init",
@@ -187,15 +195,15 @@ export class G2MirrorClient {
});
},
onTextMessage: (message: string) => {
if (this.stopped) return;
if (this.stopped || generation !== this.connectionGeneration) return;
this.handleMessage(String(message));
},
onClosed: (code: number, reason: string) => {
if (this.stopped) return;
if (this.stopped || generation !== this.connectionGeneration) return;
this.handleConnectionLost(`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""}).`);
},
onFailure: (message: string) => {
if (this.stopped) return;
if (this.stopped || generation !== this.connectionGeneration) return;
this.handleConnectionLost(`Connection failed: ${shortenError(String(message))}`);
},
});
@@ -209,6 +217,7 @@ export class G2MirrorClient {
stop(): void {
this.stopped = true;
++this.connectionGeneration;
this.rejectAllPendingLaunches("client stopped");
this.clearListRefreshTimer();
if (this.ws) {
+27
View File
@@ -0,0 +1,27 @@
/**
* Magnetic declination from Android's built-in World Magnetic Model. Declination
* is the angle from magnetic north to true north (east positive), so
* `true = magnetic + declination`. No network access is involved; the model is
* baked into the platform and is accurate to well under a degree, which is far
* inside the magnetometer's own noise.
*/
declare const android: any;
declare const global: any;
/** Declination in degrees at a place and time, or null off-Android. */
export function magneticDeclinationDegrees(
latitude: number,
longitude: number,
timestampMs: number = Date.now(),
altitudeMeters = 0,
): number | null {
if (!global.isAndroid) return null;
try {
const field = new android.hardware.GeomagneticField(latitude, longitude, altitudeMeters, timestampMs);
const declination = Number(field.getDeclination());
return Number.isFinite(declination) ? declination : null;
} catch (error) {
console.warn(`GeomagneticField failed: ${error}`);
return null;
}
}
+43 -2
View File
@@ -15,6 +15,39 @@ export function isDecodableImageFile(name: string): boolean {
return DECODABLE_IMAGE.test(name);
}
/**
* How source gray maps onto the display's 16 levels when converting to a
* GrayImage. The default (gamma 1, no dither) is a flat conversion with
* per-pixel rounding, right for UI raster such as icons and map tiles.
*/
export type GrayImageTone = {
/**
* Tone curve out = in^gamma (in 0..1). gamma > 1 darkens midtones: source
* pixels are sRGB-encoded but the G2 drives its levels roughly linearly,
* so untouched photos read washed out.
*/
gamma?: number;
/**
* Floyd-Steinberg error diffusion onto the 16 display levels instead of
* rounding each pixel independently, which bands smooth gradients.
*/
dither?: boolean;
};
/**
* Gamma for continuous-tone images (photos, album art): 2.2 undoes the sRGB
* encoding outright. Lower it toward ~1.8 if shadows crush.
*/
export const PHOTO_GAMMA = 2.2;
/** Tone preset for continuous-tone images: sRGB gamma plus dithering. */
export const PHOTO_TONE: Readonly<Required<GrayImageTone>> = { gamma: PHOTO_GAMMA, dither: true };
/** Resolve a tone to the (gamma, dither) pair the Java helpers take. */
export function resolveGrayImageTone(tone?: GrayImageTone): { gamma: number; dither: boolean } {
return { gamma: tone?.gamma ?? 1, dither: tone?.dither ?? false };
}
/**
* Decode a gray packet ([widthLo, widthHi, heightLo, heightHi, pixels...],
* the format the Java helpers return) into a GrayImage; null when empty or
@@ -34,12 +67,20 @@ export function grayImageFromPacket(raw: ArrayLike<number> | null | undefined):
/**
* Load an image file as grayscale, downscaled to fit maxWidth x maxHeight
* (aspect preserved, never upscaled); null when unreadable or undecodable.
* Pass PHOTO_TONE for continuous-tone images; the default is a flat
* conversion for UI raster.
*/
export function loadImageFileAsGray(path: string, maxWidth: number, maxHeight: number): GrayImage | null {
export function loadImageFileAsGray(
path: string,
maxWidth: number,
maxHeight: number,
tone?: GrayImageTone,
): GrayImage | null {
if (!global.isAndroid) return null;
try {
const { gamma, dither } = resolveGrayImageTone(tone);
return grayImageFromPacket(
com.faceclaw.app.ImageFileLoader.loadGray(path, Math.round(maxWidth), Math.round(maxHeight)),
com.faceclaw.app.ImageFileLoader.loadGray(path, Math.round(maxWidth), Math.round(maxHeight), gamma, dither),
);
} catch (error) {
console.warn(`loadImageFileAsGray failed for ${path}: ${error}`);
+49 -9
View File
@@ -18,6 +18,23 @@ const DIRECTIONS_ROOT = "https://api.mapbox.com/directions/v5";
const STATIC_ROOT = "https://api.mapbox.com/styles/v1";
/** Classic dark style; readable after gray conversion. Swappable for a custom Studio style later. */
const MAP_STYLE = "mapbox/dark-v11";
/**
* Render-time style override: a black background inserted beneath the road
* stack so land, landuse, water, and building fills all come out black on the
* lens (dark-v11's own fills are dark gray, which the lens shows as a lit
* haze). Static Images allows one added layer per request; `before_layer`
* must name a layer in the style, so candidates are tried in order and a
* rejected one is skipped for the rest of the session, ending with the plain
* style (the worker's levelling pass then does what it can client-side).
*/
const STYLE_OVERRIDE_LAYER = JSON.stringify({
id: "faceclaw-blackout",
type: "background",
paint: { "background-color": "#000000" },
});
/** First road layer of dark-v11, then a layer common to every Streets-based style. */
const STYLE_OVERRIDE_BEFORE_LAYERS = ["tunnel-path-trail", "building"];
let styleOverrideIndex = 0;
const FETCH_TIMEOUT_MS = 15_000;
export type RouteProfile = "driving" | "walking" | "cycling";
@@ -188,18 +205,32 @@ export async function fetchStaticMapGray(options: {
: `${round6(options.camera.longitude)},${round6(options.camera.latitude)},` +
`${options.camera.zoom.toFixed(2)},${Math.round(normalizeBearing(options.camera.bearingDeg))}`;
const size = `${Math.round(options.width)}x${Math.round(options.height)}`;
const params = queryString({
access_token: requireToken(),
logo: "false",
attribution: "false",
});
const url = `${STATIC_ROOT}/${MAP_STYLE}/static/${overlay}${camera}/${size}?${params}`;
let bytes: ArrayBuffer;
for (;;) {
const beforeLayer = STYLE_OVERRIDE_BEFORE_LAYERS[styleOverrideIndex];
const params: Record<string, string> = {
access_token: requireToken(),
logo: "false",
attribution: "false",
};
if (beforeLayer !== undefined) {
params.addlayer = STYLE_OVERRIDE_LAYER;
params.before_layer = beforeLayer;
}
const url = `${STATIC_ROOT}/${MAP_STYLE}/static/${overlay}${camera}/${size}?${queryString(params)}`;
const response = await fetchWithTimeout(url, "Mapbox static map");
if (!response.ok) {
const response = await fetchWithTimeout(url, "Mapbox static map");
if (response.ok) {
bytes = await response.arrayBuffer();
break;
}
if (beforeLayer !== undefined && isStyleOverrideRejection(response.status)) {
console.warn(`Mapbox rejected the map style override (before_layer=${beforeLayer}, HTTP ${response.status}); falling back`);
styleOverrideIndex++;
continue;
}
throw new Error(`Mapbox static map failed (HTTP ${response.status}).`);
}
const bytes = await response.arrayBuffer();
if (!global.isAndroid) throw new Error("Map decoding is only available on Android.");
const image = grayImageFromPacket(
com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(bytes, Math.round(options.width), Math.round(options.height)),
@@ -281,6 +312,15 @@ async function fetchWithTimeout(url: string, label: string): Promise<Response> {
}
}
/**
* A client error that plausibly blames the added layer (an unknown
* before_layer, or a style without it), as opposed to token or quota trouble
* that the plain request would hit just the same.
*/
function isStyleOverrideRejection(status: number): boolean {
return status >= 400 && status < 500 && status !== 401 && status !== 403 && status !== 429;
}
function normalizeBearing(bearingDeg: number): number {
return ((bearingDeg % 360) + 360) % 360;
}
+83
View File
@@ -0,0 +1,83 @@
import { getStringSetting, setStringSetting } from "./settings-store";
export const MEDIA_APPS_KEY = "music.apps";
// Exact package IDs only: browsers, social feeds and messaging apps commonly
// publish sessions for embedded clips. Users can explicitly enable any of them.
export const DEFAULT_IGNORED_MEDIA_APPS: readonly string[] = [
"com.android.chrome",
"org.mozilla.firefox",
"com.microsoft.emmx",
"com.sec.android.app.sbrowser",
"com.facebook.katana",
"com.facebook.orca",
"com.instagram.android",
"com.whatsapp",
"org.telegram.messenger",
"com.snapchat.android",
"com.reddit.frontpage",
"com.twitter.android",
"com.discord",
];
export type MediaApp = {
packageName: string;
appName: string;
/** Absent until the user chooses; built-in defaults can evolve independently. */
enabled?: boolean;
};
/** Keep discovered apps after their sessions end, including ignored apps. */
export function readMediaApps(): MediaApp[] {
try {
const parsed = JSON.parse(getStringSetting(MEDIA_APPS_KEY, "[]"));
if (!Array.isArray(parsed)) return [];
const apps = new Map<string, MediaApp>();
for (const app of parsed) {
if (!app || typeof app.packageName !== "string" || !app.packageName) continue;
apps.set(app.packageName, {
packageName: app.packageName,
appName: typeof app.appName === "string" && app.appName ? app.appName : app.packageName,
...(typeof app.enabled === "boolean" ? { enabled: app.enabled } : {}),
});
}
return Array.from(apps.values());
} catch {
return [];
}
}
export function rememberMediaApps(discovered: { packageName: string; appName: string }[]): void {
const apps = new Map(readMediaApps().map((app) => [app.packageName, app]));
let changed = false;
for (const app of discovered) {
if (!app.packageName) continue;
const previous = apps.get(app.packageName);
const appName = app.appName || previous?.appName || app.packageName;
if (!previous || previous.appName !== appName) {
apps.set(app.packageName, { ...previous, packageName: app.packageName, appName });
changed = true;
}
}
if (changed) setStringSetting(MEDIA_APPS_KEY, JSON.stringify(Array.from(apps.values())));
}
export function ignoredMediaPackages(): string[] {
const ignored = new Set(DEFAULT_IGNORED_MEDIA_APPS);
for (const app of readMediaApps()) {
if (app.enabled === true) ignored.delete(app.packageName);
else if (app.enabled === false) ignored.add(app.packageName);
}
return Array.from(ignored).sort();
}
export function isMediaAppEnabled(packageName: string): boolean {
return !ignoredMediaPackages().includes(packageName);
}
export function setMediaAppEnabled(app: MediaApp, enabled: boolean): void {
if (!app.packageName) return;
const apps = new Map(readMediaApps().map((item) => [item.packageName, item]));
apps.set(app.packageName, { packageName: app.packageName, appName: app.appName || app.packageName, enabled });
setStringSetting(MEDIA_APPS_KEY, JSON.stringify(Array.from(apps.values())));
}
+24 -2
View File
@@ -1,4 +1,6 @@
import { Utils } from "@nativescript/core";
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
import { onSettingsStoreChanged } from "./settings-store";
declare const com: any;
@@ -34,11 +36,12 @@ export class FaceclawMediaBrowserBridge {
private readonly pendingConnects = new Map<number, PendingRequest<string>>();
private readonly pendingBrowses = new Map<number, PendingRequest<MediaBrowseItem[]>>();
private appsCache: MediaBrowserApp[] | null = null;
private connectedPackage = "";
/** Installed apps exposing a media browser service; cached after the first call. */
listBrowsableApps(refresh = false): MediaBrowserApp[] {
if (!global.isAndroid) return [];
if (this.appsCache && !refresh) return this.appsCache;
if (this.appsCache && !refresh) return this.enabledApps();
this.ensureBrowser();
if (!this.browser) return [];
try {
@@ -52,11 +55,17 @@ export class FaceclawMediaBrowserBridge {
serviceClass: String(app.serviceClass ?? ""),
appName: String(app.appName ?? ""),
}));
rememberMediaApps(this.appsCache);
} catch (error) {
console.warn(`media browser app list parse failed: ${error}`);
this.appsCache = [];
}
return this.appsCache;
return this.enabledApps();
}
private enabledApps(): MediaBrowserApp[] {
const ignored = new Set(ignoredMediaPackages());
return (this.appsCache ?? []).filter((app) => !ignored.has(app.packageName));
}
/**
@@ -65,9 +74,11 @@ export class FaceclawMediaBrowserBridge {
*/
connect(app: MediaBrowserApp): Promise<string> {
if (!global.isAndroid) return Promise.reject(new Error("Android only"));
if (!isMediaAppEnabled(app.packageName)) return Promise.reject(new Error("App is ignored in Music settings"));
this.ensureBrowser();
if (!this.browser) return Promise.reject(new Error("Media browser unavailable"));
const requestId = this.nextRequestId++;
this.connectedPackage = app.packageName;
return new Promise<string>((resolve, reject) => {
this.pendingConnects.set(requestId, { resolve, reject });
this.browser.connect(requestId, app.packageName, app.serviceClass);
@@ -77,6 +88,10 @@ export class FaceclawMediaBrowserBridge {
/** Fetch one level of the connected service's content tree. */
browse(parentId: string): Promise<MediaBrowseItem[]> {
if (!global.isAndroid) return Promise.reject(new Error("Android only"));
if (this.connectedPackage && !isMediaAppEnabled(this.connectedPackage)) {
this.disconnect();
return Promise.reject(new Error("App is ignored in Music settings"));
}
this.ensureBrowser();
if (!this.browser) return Promise.reject(new Error("Media browser unavailable"));
const requestId = this.nextRequestId++;
@@ -89,6 +104,7 @@ export class FaceclawMediaBrowserBridge {
/** Start playback of a browsed item through the connected player's session. */
playFromMediaId(mediaId: string): void {
if (!global.isAndroid) return;
if (!this.connectedPackage || !isMediaAppEnabled(this.connectedPackage)) return;
this.browser?.playFromMediaId(mediaId);
}
@@ -96,6 +112,7 @@ export class FaceclawMediaBrowserBridge {
disconnect(): void {
if (!global.isAndroid) return;
this.rejectAllPending("Disconnected");
this.connectedPackage = "";
this.browser?.disconnect();
}
@@ -115,6 +132,11 @@ export class FaceclawMediaBrowserBridge {
throw new Error("Android application context unavailable");
}
this.browser = new com.faceclaw.app.FaceclawMediaBrowser(context);
onSettingsStoreChanged((key) => {
if (key === MEDIA_APPS_KEY && this.connectedPackage && !isMediaAppEnabled(this.connectedPackage)) {
this.disconnect();
}
});
this.listenerProxy = new com.faceclaw.app.FaceclawMediaBrowserListener({
onConnectResult: (requestId: number, connected: boolean, rootId: string, error: string) => {
const pending = this.pendingConnects.get(Number(requestId));
+22 -13
View File
@@ -1,17 +1,11 @@
import { Utils } from "@nativescript/core";
import { GrayImage } from "../graphics/image";
import { grayImageFromPacket } from "./image-files";
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
import { ignoredMediaPackages, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
import { onSettingsStoreChanged } from "./settings-store";
declare const com: any;
/**
* Tone curve applied to album art before it is dithered down to the display's
* 16 gray levels (out = in^gamma, in 0..1). Cover art is sRGB-encoded but the
* G2 drives its levels roughly linearly, so untouched art reads washed out;
* 2.2 undoes the sRGB encoding. Lower it toward ~1.8 if shadows crush.
*/
const ALBUM_ART_GAMMA = 2.2;
export type MediaQueueItem = {
id: number;
title: string;
@@ -141,16 +135,16 @@ export class FaceclawMediaControllerBridge {
}
/**
* Album art for the current item, grayscale (gamma-corrected and dithered
* onto the display's 16 levels), scaled to fit maxSize; null when the
* player provides none.
* Album art for the current item, grayscale with the shared photo tone
* (gamma-corrected and dithered onto the display's 16 levels), scaled to
* fit maxSize; null when the player provides none.
*/
getAlbumArt(maxSize: number): GrayImage | null {
if (!global.isAndroid) return null;
this.ensureController();
if (!this.controller) return null;
return grayImageFromPacket(
this.controller.getAlbumArtGray(Math.round(maxSize), ALBUM_ART_GAMMA),
this.controller.getAlbumArtGray(Math.round(maxSize), PHOTO_TONE.gamma, PHOTO_TONE.dither),
);
}
@@ -181,7 +175,22 @@ export class FaceclawMediaControllerBridge {
throw new Error("Android application context unavailable");
}
this.controller = new com.faceclaw.app.FaceclawMediaController(context);
// Install defaults and saved overrides before starting session discovery.
this.controller.setIgnoredPackagesJson(JSON.stringify(ignoredMediaPackages()));
onSettingsStoreChanged((key) => {
if (key === MEDIA_APPS_KEY) {
this.controller.setIgnoredPackagesJson(JSON.stringify(ignoredMediaPackages()));
}
});
this.listenerProxy = new com.faceclaw.app.FaceclawMediaControllerListener({
onSessionAppsChanged: (appsJson: string) => {
try {
const apps = JSON.parse(String(appsJson));
rememberMediaApps(apps);
} catch (error) {
console.warn(`media session apps parse failed: ${error}`);
}
},
onStateChange: (
playbackState: string,
packageName: string,
+13 -9
View File
@@ -1,4 +1,4 @@
import { loadNightscoutSettings, nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings";
import { nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings";
import { fetchWithUserAgent } from "../util/http";
export type NightscoutPoint = {
@@ -33,6 +33,10 @@ export type NightscoutState = {
iob: number | null;
cob: number | null;
cageTimestampMs: number | null;
reservoirUnits: number | null;
batteryVoltage: number | null;
loopTimestampMs: number | null;
/** Pump communication age and operating state; measurements are separate. */
pumpStatus: string;
openapsStatusShort: string;
carbs: NightscoutCarbEvent[];
@@ -121,6 +125,9 @@ const DEFAULT_NIGHTSCOUT_STATE: NightscoutState = {
iob: null,
cob: null,
cageTimestampMs: null,
reservoirUnits: null,
batteryVoltage: null,
loopTimestampMs: null,
pumpStatus: "",
openapsStatusShort: "--",
carbs: [],
@@ -259,6 +266,9 @@ export class NightscoutBridge {
iob: Number.isFinite(iob) ? iob : null,
cob,
cageTimestampMs: latestSiteChange > 0 ? latestSiteChange : null,
reservoirUnits: firstFiniteNumber(latestDeviceStatus?.pump?.reservoir),
batteryVoltage: firstFiniteNumber(latestDeviceStatus?.pump?.battery?.voltage),
loopTimestampMs: loopTimestampMs > 0 ? loopTimestampMs : null,
pumpStatus,
openapsStatusShort: loopTimestampMs > 0 ? formatElapsedShort(nowMs - loopTimestampMs) : "--",
carbs: parsedTreatments.carbs,
@@ -359,6 +369,8 @@ function parseTimestampMs(value: unknown): number {
function firstFiniteNumber(...values: unknown[]): number | null {
for (const value of values) {
if (typeof value !== "number" && typeof value !== "string") continue;
if (typeof value === "string" && !value.trim()) continue;
const numeric = Number(value);
if (Number.isFinite(numeric)) {
return numeric;
@@ -371,8 +383,6 @@ function formatPumpStatus(status: NightscoutDeviceStatusResponse | undefined, no
if (!status?.pump) {
return "--";
}
const reservoir = firstFiniteNumber(status.pump.reservoir);
const voltage = firstFiniteNumber(status.pump.battery?.voltage);
const commTimestampMs = Math.max(
parseTimestampMs(status.pump.status?.timestamp),
parseTimestampMs(status.pump.clock),
@@ -380,12 +390,6 @@ function formatPumpStatus(status: NightscoutDeviceStatusResponse | undefined, no
);
const stateLabel = summarizePumpState(status);
const parts: string[] = [];
if (reservoir !== null) {
parts.push(`${Math.round(reservoir)}U`);
}
if (voltage !== null) {
parts.push(`${voltage.toFixed(2)}v`);
}
if (commTimestampMs > 0) {
parts.push(`${formatElapsedShort(nowMs - commTimestampMs)} ago`);
}
+6 -1
View File
@@ -1,10 +1,13 @@
import { GrayImage } from "../graphics/image";
import { logCurrent, spanCurrent } from "./frame-timings";
import { toUint8Array } from "../util/array-util";
import { rememberNotificationSources } from "./notification-sources";
declare const com: any;
const ICON_SIZE = 24;
/** Ask the native listener for every active source, including those below the list's display limit. */
export const ALL_NOTIFICATIONS = 0x7fffffff;
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
// notification is posted or dismissed (see invalidateIconCaches callers), so
// the tray is kept fresh by invalidation, not by expiry. A short TTL just
@@ -158,7 +161,9 @@ export function readActiveNotifications(maxNotifications = 50): AndroidNotificat
);
const parsed = JSON.parse(json);
if (!Array.isArray(parsed)) return [];
return parsed.map(normalizeNotification).filter((item): item is AndroidNotification => Boolean(item));
const notifications = parsed.map(normalizeNotification).filter((item): item is AndroidNotification => Boolean(item));
rememberNotificationSources(notifications);
return notifications;
} catch {
return [];
}
+59
View File
@@ -0,0 +1,59 @@
import { getStringSetting, setStringSetting } from "./settings-store";
const SOURCES_KEY = "notifications.sources";
export type NotificationSource = {
packageName: string;
appName: string;
showOnGlasses: boolean;
};
/** Sources outlive their active notifications so a muted app can always be re-enabled. */
export function readNotificationSources(): NotificationSource[] {
try {
const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]"));
if (!Array.isArray(parsed)) return [];
return parsed.filter((source) => source && typeof source.packageName === "string" && source.packageName)
.map((source) => ({
packageName: source.packageName,
appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName,
showOnGlasses: source.showOnGlasses !== false,
}));
} catch {
return [];
}
}
export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void {
const sources = new Map(readNotificationSources().map((source) => [source.packageName, source]));
let changed = false;
for (const notification of notifications) {
if (!notification.packageName) continue;
const previous = sources.get(notification.packageName);
const appName = notification.appName || previous?.appName || notification.packageName;
if (!previous || previous.appName !== appName) {
sources.set(notification.packageName, {
packageName: notification.packageName,
appName,
showOnGlasses: previous?.showOnGlasses ?? true,
});
changed = true;
}
}
if (changed) setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
}
export function shouldShowNotificationOnGlasses(packageName: string): boolean {
return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses ?? true;
}
export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void {
if (!source.packageName) return;
const sources = new Map(readNotificationSources().map((item) => [item.packageName, item]));
sources.set(source.packageName, {
packageName: source.packageName,
appName: source.appName || source.packageName,
showOnGlasses: enabled,
});
setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
}
+74 -16
View File
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
// Base64 audio (or the commit sentinel) queued until the socket opens.
private readonly pendingChunks: string[] = [];
private readonly upsampler = new PcmUpsampler();
private latestText = "";
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
private readonly commits: boolean[] = [];
private committed = false;
constructor(private readonly options: OpenAiSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`OpenAI connection failed: ${String(message)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
},
});
try {
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
);
this.options.onStatus("Connecting to OpenAI...");
} catch (error) {
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: commit the buffer for a final transcript. */
finish(): void {
this.committed = true;
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
this.commits.push(paragraphBreakAfter);
if (this.open) {
this.sendChunk("__commit__");
} else {
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.committed = true;
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
return;
}
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("OpenAI send failed.");
}
} catch (error) {
console.warn("openai send failed", error);
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
}
}
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
case "session.updated":
this.options.onStatus("Listening (OpenAI)...");
return;
case "conversation.item.input_audio_transcription.delta":
this.latestText += String(message.delta ?? "");
this.options.onTranscript({ text: this.latestText, isFinal: false });
case "input_audio_buffer.committed": {
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
return;
case "conversation.item.input_audio_transcription.completed":
this.latestText = String(message.transcript ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
// The utterance is done; nothing further is coming after the commit.
if (this.committed) this.stop();
}
case "conversation.item.input_audio_transcription.delta": {
this.item(String(message.item_id)).text += String(message.delta ?? "");
this.emitItems();
return;
}
case "conversation.item.input_audio_transcription.completed": {
const item = this.item(String(message.item_id));
item.text = String(message.transcript ?? item.text);
item.final = true;
this.emitItems();
if (this.committed && !this.items.size && !this.commits.length) this.stop();
return;
}
case "error":
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
return;
}
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
return;
default:
return;
}
}
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
let item = this.items.get(id);
if (!item) {
item = { text: "", final: false, paragraphBreakAfter: false };
this.items.set(id, item);
}
return item;
}
private emitItems(): void {
// Committed notifications precede transcription events, so Map insertion
// order is audio order even when completion events arrive out of order.
for (const [id, item] of this.items) {
if (!item.final) break;
this.items.delete(id);
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
}
if (this.items.size) {
const items = [...this.items.values()];
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
const transcribeText = items.map((item, index) =>
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
).join("");
this.options.onTranscript({ text, transcribeText, isFinal: false });
}
}
}
function sessionConfig(): object {
+30
View File
@@ -0,0 +1,30 @@
import { Application, isAndroid } from "@nativescript/core";
import { getStringSetting, onSettingsStoreChanged } from "./settings-store";
// Shared with phoneRotationSetting; keep startup independent of the glasses UI.
const ROTATION_KEY = "phone.rotation";
/** Installed once on the UI thread; also covers navigation and activity recreation. */
export function registerPhoneRotation(): void {
if (!isAndroid) return;
const apply = (activity = Application.android.foregroundActivity ?? Application.android.startActivity): void => {
if (!activity) return;
const orientations = android.content.pm.ActivityInfo;
const requested = {
auto: orientations.SCREEN_ORIENTATION_UNSPECIFIED,
portrait: orientations.SCREEN_ORIENTATION_PORTRAIT,
landscape: orientations.SCREEN_ORIENTATION_SENSOR_LANDSCAPE,
}[getStringSetting(ROTATION_KEY, "auto")] ?? orientations.SCREEN_ORIENTATION_UNSPECIFIED;
if (activity.getRequestedOrientation() !== requested) {
activity.setRequestedOrientation(requested);
}
};
Application.android.on(Application.android.activityCreatedEvent, (args) => apply(args.activity));
Application.android.on(Application.android.activityResumedEvent, (args) => apply(args.activity));
onSettingsStoreChanged((key) => {
if (key === ROTATION_KEY) apply();
});
apply();
}
+6 -1
View File
@@ -1,5 +1,5 @@
import { ImageSource, Utils } from "@nativescript/core";
import type { FaceclawCommunicatorBridge, SurfaceOptions } from "./faceclaw-communicator";
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
declare const com: any;
declare const java: any;
@@ -19,6 +19,7 @@ export type DisplayTarget = Pick<
| "configureSurface"
| "removeSurface"
| "setSurfaceVisible"
| "setUnderlayDim"
| "setScreenBlanked"
| "submitSurfaceFrame"
| "waitForFrameFinished"
@@ -96,6 +97,10 @@ export class PreviewDisplayTarget implements DisplayTarget {
this.compositor.setSurfaceVisible(id, Boolean(visible));
}
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
this.compositor.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
}
async setScreenBlanked(blanked: boolean): Promise<void> {
this.compositor.setScreenBlanked(Boolean(blanked));
}
+141
View File
@@ -0,0 +1,141 @@
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
import { SpeechPauseDetector } from "./speech-pause";
// Ten seconds of unsent PCM. Never replay audio already sent: that would
// duplicate words already shown. An extended outage drops the oldest audio.
const MAX_PENDING_BYTES = 16000 * 2 * 10;
/** Owns retries and audio buffering independently of the provider protocol. */
export class ReconnectingSttClient implements CloudSttClient {
private client: CloudSttClient | null = null;
private generation = 0;
private stopped = false;
private finishing = false;
private ready = false;
private retry: ReturnType<typeof setTimeout> | null = null;
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
private attempt = 0;
private pending: Uint8Array[] = [];
private pendingBytes = 0;
private latest: CloudSttTranscriptEvent | null = null;
private readonly pause = new SpeechPauseDetector();
constructor(
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
) {}
start(): void {
if (this.stopped || this.client || this.retry !== null) return;
const generation = ++this.generation;
const current = () => !this.stopped && generation === this.generation;
this.client = this.create({
...this.options,
onReady: () => {
if (!current()) return;
this.ready = true;
const chunks = this.pending;
this.pending = [];
this.pendingBytes = 0;
for (const chunk of chunks) this.acceptPcm(chunk);
if (this.finishing) this.client?.finish();
},
onTranscript: (event) => {
if (!current()) return;
// A handshake alone can immediately be followed by another failure.
this.attempt = 0;
this.latest = event.isFinal ? null : event;
this.options.onTranscript(event);
},
onStatus: (status) => { if (current()) this.options.onStatus(status); },
onError: (message) => {
if (!current()) return;
this.stop();
this.options.onError(message);
},
onDisconnected: (message) => {
if (current()) this.disconnected(message);
},
});
try {
this.client.start();
} catch (error) {
this.disconnected(`Transcription connection failed: ${String(error)}`);
}
}
acceptPcm(pcm: Uint8Array): void {
if (this.stopped || !pcm.length) return;
if (!this.ready) {
const copy = pcm.slice(-MAX_PENDING_BYTES);
this.pending.push(copy);
this.pendingBytes += copy.length;
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
return;
}
this.client?.acceptPcm(pcm);
// A rejected send synchronously reports disconnection. Keep that packet
// with the unsent audio instead of losing it at the outage boundary.
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
}
finish(): void {
if (this.stopped || this.finishing) return;
this.finishing = true;
// Releasing the last mic holder must not leave a reconnect loop alive.
if (this.retry !== null) {
this.preservePartial();
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), 5000);
if (this.ready) this.client?.finish();
}
stop(): void {
this.stopped = true;
++this.generation;
if (this.retry !== null) clearTimeout(this.retry);
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
this.retry = this.finishTimeout = null;
this.client?.stop();
this.client = null;
this.ready = false;
this.pending = [];
this.pendingBytes = 0;
this.latest = null;
}
private preservePartial(): void {
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
this.latest = null;
}
private disconnected(message: string): void {
if (this.stopped) return;
++this.generation; // Ignore late finals / closes from the abandoned socket.
this.ready = false;
this.client?.stop();
this.client = null;
this.pause.reset();
this.preservePartial();
if (this.stopped) return; // A transcript listener can release capture.
if (/\b(401|402|403)\b/.test(message)) {
this.stop();
this.options.onError(message);
return;
}
if (this.finishing) {
this.stop();
return;
}
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
this.retry = setTimeout(() => {
this.retry = null;
this.start();
}, delay);
}
}
+48 -13
View File
@@ -1,4 +1,5 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
import { TimedTranscript } from "./transcript-format";
declare const com: any;
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
// PCM queued until the socket opens and the config message is sent.
private readonly pendingPcm: Uint8Array[] = [];
private pendingFinish = false;
private finishing = false;
// Concatenation of all final tokens so far.
private finalText = "";
private formatted = new TimedTranscript();
constructor(private readonly options: SonioxSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
// The API key rides in the config message; there is no auth header.
this.trySendText(
JSON.stringify({
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
audio_format: "pcm_s16le",
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
}),
);
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingPcm.splice(0)) {
this.sendPcm(chunk);
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
this.finish();
}
this.options.onStatus("Listening (Soniox)...");
this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`Soniox connection failed: ${String(message)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
},
});
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
/** End of utterance: an empty text frame tells Soniox the audio is done. */
finish(): void {
if (this.closed) return;
this.finishing = true;
if (this.open) {
this.trySendText("");
} else {
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
}
}
commitSegment(): void {
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingPcm.length = 0;
}
private sendPcm(pcm: Uint8Array): void {
try {
this.ws?.sendBinary(toJavaBytes(pcm));
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
this.options.onDisconnected?.("Soniox audio send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
private trySendText(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("Soniox send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
return;
}
if (message?.error_code != null) {
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
else this.options.onError(error);
return;
}
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
let nonFinal = "";
let preview: TimedTranscript | null = null;
for (const token of tokens) {
const tokenText = String(token?.text ?? "");
// Markers emitted by endpoint detection / manual finalize; not speech.
if (tokenText === "<end>" || tokenText === "<fin>") continue;
if (tokenText === "<end>" || tokenText === "<fin>") {
if (token?.is_final && this.finalText) {
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
this.finalText = "";
this.formatted = new TimedTranscript();
}
continue;
}
if (token?.is_final) {
this.finalText += tokenText;
this.formatted.append({ ...token, text: tokenText });
} else {
nonFinal += tokenText;
preview ??= this.formatted.copy();
preview.append({ ...token, text: tokenText });
}
}
if (message?.finished) {
this.options.onTranscript({ text: this.finalText, isFinal: true });
this.stop();
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
if (this.finishing) this.stop();
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
return;
}
if (tokens.length > 0) {
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
}
}
}
+44
View File
@@ -0,0 +1,44 @@
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
export const TRANSCRIPT_PAUSE_MS = 1500;
export class SpeechPauseDetector {
private noiseFloor = 220;
private speaking = false;
private quietSamples = 0;
reset(): void {
this.noiseFloor = 220;
this.speaking = false;
this.quietSamples = 0;
}
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
accept(pcm: Uint8Array): boolean {
const count = pcm.length >> 1;
if (!count) return false;
let squares = 0;
for (let i = 0; i < count; i++) {
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
squares += sample * sample;
}
const rms = Math.sqrt(squares / count);
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
if (rms >= threshold) {
this.speaking = true;
this.quietSamples = 0;
} else {
// Slowly follow background levels without learning speech as noise.
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
if (this.speaking) {
this.quietSamples += count;
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
this.speaking = false;
this.quietSamples = 0;
return true;
}
}
}
return false;
}
}
-29
View File
@@ -1,29 +0,0 @@
import { Utils } from "@nativescript/core";
declare const com: any;
function getContext(): android.content.Context | null {
if (!global.isAndroid) return null;
return Utils.android.getApplicationContext() ?? null;
}
/** Schedule the durable Android alarm backing one on-glasses timer. */
export function scheduleTimerNotification(timerId: number, triggerAtMs: number, durationLabel: string): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawTimerNotifications.schedule(context, timerId, triggerAtMs, durationLabel);
}
/** Cancel both a pending alarm and any already-posted notification. */
export function cancelTimerNotification(timerId: number): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawTimerNotifications.cancel(context, timerId);
}
/** Prompt worker-side expiry path; Java deduplicates it against the alarm. */
export function fireTimerNotification(timerId: number, durationLabel: string): void {
const context = getContext();
if (!context) return;
com.faceclaw.app.FaceclawTimerNotifications.fireNow(context, timerId, durationLabel);
}
+32
View File
@@ -0,0 +1,32 @@
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
export class TimedTranscript {
text = "";
private endMs: number | undefined;
private speaker: string | undefined;
copy(): TimedTranscript {
const copy = new TimedTranscript();
copy.text = this.text;
copy.endMs = this.endMs;
copy.speaker = this.speaker;
return copy;
}
append(token: TranscriptToken): void {
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
} else {
this.text += token.text;
}
if (token.text.trim()) {
if (token.end_ms != null) this.endMs = token.end_ms;
if (token.speaker != null) this.speaker = token.speaker;
}
}
}
+28 -9
View File
@@ -1,6 +1,8 @@
import { Utils } from "@nativescript/core";
import { CloudSttClient } from "./cloud-stt";
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SpeechPauseDetector } from "./speech-pause";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
@@ -14,7 +16,7 @@ export type VoiceControlState = {
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = {
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
* Complete best transcript of the current utterance. REPLACE semantics —
* render as-is, replacing any previous partial. Not a delta.
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private readonly speechPauseListeners = new Set<() => void>();
private readonly speechPause = new SpeechPauseDetector();
private readonly speechEndListeners = new Set<() => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
return () => this.speechEndListeners.delete(listener);
}
/** Pauses in on-device recognition, for Transcribe's presentation only. */
onSpeechPause(listener: () => void): () => void {
this.speechPauseListeners.add(listener);
return () => this.speechPauseListeners.delete(listener);
}
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
this.controller?.clearSpeakerVerification();
}
this.speechPause.reset();
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
if (cloudClient) {
this.cloudClient = cloudClient;
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
if (options.provider === "onboard") return null;
const sttOptions = {
apiKey: "",
onTranscript: (event: { text: string; isFinal: boolean }) =>
this.emitTranscript(event.text, event.isFinal),
onTranscript: (event: CloudSttTranscriptEvent) =>
this.emitTranscript(event.text, event.isFinal, event),
onStatus: (status: string) => this.setStatus(status),
onError: (message: string) => this.setStatus(message),
};
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
if (options.provider === "elevenlabs") {
const apiKey = options.elevenLabsApiKey.trim();
if (!apiKey) {
this.setStatus("No ElevenLabs key set; using on-device voice.");
return null;
}
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return new SonioxSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new SonioxSttClient(config), apiKey);
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
return null;
}
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
}
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
this.controller.setListener(this.listenerProxy);
}
private emitTranscript(text: string, isFinal: boolean): void {
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
// "My voice only": a rejected session's final transcript is emptied so
// nothing downstream (assistant, dictation) acts on another speaker's
// words. The verification verdict is posted before the final, and cloud
// finals arrive over the network later still, so the flag is settled.
if (isFinal && this.verificationActive && this.verificationRejected) {
text = "";
metadata = undefined;
}
const event = { text, isFinal };
const event = { ...metadata, text, isFinal };
for (const listener of this.transcriptListeners) {
listener(event);
}