forked from Blendman974/faceclaw
Merge branch 'main' into ble-bandwidth-test
This commit is contained in:
commit
50cbc3a756
198 files changed
+15988
-2899
No files matched your search
@@ -0,0 +1,154 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Bridge to the Java alarm clock (FaceclawAlarms / FaceclawAlarmService):
|
||||
* the durable phone side of the Timers app's timers and alarms. The phone
|
||||
* schedules an exact alarm-clock alarm per item, re-arms them after reboots
|
||||
* and time changes, and when one comes due shows it on the phone (silently
|
||||
* at first) and adds phone sound if the glasses cannot carry it. Ids are the
|
||||
* engine's item ids.
|
||||
*/
|
||||
|
||||
export type AlarmKind = "timer" | "alarm";
|
||||
|
||||
export type PhoneAlarmAction = { id: number; action: "dismiss" | "snooze"; minutes: number; atMs: number };
|
||||
|
||||
export type AlarmReliabilityIssue = { code: string; message: string; fixable: boolean };
|
||||
|
||||
function getContext(): android.content.Context | null {
|
||||
if (!global.isAndroid) return null;
|
||||
return Utils.android.getApplicationContext() ?? null;
|
||||
}
|
||||
|
||||
/** Schedule (or move) the phone alarm that rings `title` / `text` at triggerAtMs. */
|
||||
export function schedulePhoneAlarm(
|
||||
itemId: number,
|
||||
triggerAtMs: number,
|
||||
title: string,
|
||||
text: string,
|
||||
kind: AlarmKind,
|
||||
snoozeMinutes: number,
|
||||
): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.schedule(context, itemId, triggerAtMs, title, text, kind, snoozeMinutes);
|
||||
}
|
||||
|
||||
/** Cancel the pending phone alarm, stop it if ringing, and clear its notification. */
|
||||
export function cancelPhoneAlarm(itemId: number): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.cancel(context, itemId);
|
||||
}
|
||||
|
||||
/** The engine's own expiry path: start the phone ringing flow (deduplicated against the alarm's). */
|
||||
export function ringPhoneAlarm(itemId: number, title: string, text: string, kind: AlarmKind, snoozeMinutes: number): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.ring(context, itemId, title, text, kind, snoozeMinutes);
|
||||
}
|
||||
|
||||
/** The glasses are showing the item: the phone starts its 30-second acknowledgement clock. */
|
||||
export function phoneAlarmDeliveredToGlasses(itemId: number): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.deliveredToGlasses(context, itemId);
|
||||
}
|
||||
|
||||
/** The wearer dealt with the item on the glasses: the phone goes quiet. */
|
||||
export function acknowledgePhoneAlarm(itemId: number): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.acknowledge(context, itemId);
|
||||
}
|
||||
|
||||
/** Tell the phone whether the glasses could carry an alarm right now. */
|
||||
export function setPhoneAlarmGlassesStatus(connected: boolean, worn: boolean, charging: boolean): void {
|
||||
if (!global.isAndroid) return;
|
||||
com.faceclaw.app.FaceclawAlarms.setGlassesStatus(connected, worn, charging);
|
||||
}
|
||||
|
||||
let retainedListenerProxy: any = null;
|
||||
const actionListeners = new Set<(action: PhoneAlarmAction) => void>();
|
||||
|
||||
/** Live phone-side dismiss / snooze events (journaled ones arrive through drainPhoneAlarmJournal). */
|
||||
export function onPhoneAlarmAction(listener: (action: PhoneAlarmAction) => void): () => void {
|
||||
if (retainedListenerProxy === null && global.isAndroid) {
|
||||
retainedListenerProxy = new com.faceclaw.app.FaceclawAlarmListener({
|
||||
onPhoneAction: (id: number, action: string, minutes: number) => {
|
||||
const event: PhoneAlarmAction = {
|
||||
id: Number(id),
|
||||
action: String(action) === "snooze" ? "snooze" : "dismiss",
|
||||
minutes: Number(minutes),
|
||||
atMs: Date.now(),
|
||||
};
|
||||
for (const registered of Array.from(actionListeners)) {
|
||||
try {
|
||||
registered(event);
|
||||
} catch (error) {
|
||||
console.warn(`phone alarm action listener failed: ${error}`);
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
com.faceclaw.app.FaceclawAlarms.setListener(retainedListenerProxy);
|
||||
}
|
||||
actionListeners.add(listener);
|
||||
return () => {
|
||||
actionListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
/** Phone-side actions taken while the engine was not listening, oldest first; cleared on read. */
|
||||
export function drainPhoneAlarmJournal(): PhoneAlarmAction[] {
|
||||
const context = getContext();
|
||||
if (!context) return [];
|
||||
let parsed: any;
|
||||
try {
|
||||
parsed = JSON.parse(String(com.faceclaw.app.FaceclawAlarms.drainJournal(context)));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed
|
||||
.filter((entry) => entry && typeof entry === "object" && Number.isFinite(Number(entry.id)))
|
||||
.map((entry) => ({
|
||||
id: Number(entry.id),
|
||||
action: entry.action === "snooze" ? "snooze" : "dismiss",
|
||||
minutes: Number(entry.minutes) || 0,
|
||||
atMs: Number(entry.at) || 0,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Conditions under which the phone may fail to ring, most serious first. */
|
||||
export function checkPhoneAlarmReliability(): AlarmReliabilityIssue[] {
|
||||
const context = getContext();
|
||||
if (!context) return [];
|
||||
let parsed: any;
|
||||
try {
|
||||
parsed = JSON.parse(String(com.faceclaw.app.FaceclawAlarms.checkReliability(context)));
|
||||
} catch (error) {
|
||||
console.warn(`alarm reliability check failed: ${error}`);
|
||||
return [];
|
||||
}
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed
|
||||
.filter((entry) => entry && typeof entry === "object" && typeof entry.code === "string")
|
||||
.map((entry) => ({ code: entry.code, message: String(entry.message ?? ""), fixable: entry.fixable !== false }));
|
||||
}
|
||||
|
||||
/** Open the system screen where a reliability issue can be fixed. */
|
||||
export function openPhoneAlarmReliabilityFix(code: string): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawAlarms.openReliabilityFix(context, code);
|
||||
}
|
||||
|
||||
/** The phone's ring / miss / action log, one line per entry, newest last. */
|
||||
export function readPhoneAlarmLog(): string {
|
||||
const context = getContext();
|
||||
if (!context) return "";
|
||||
return String(com.faceclaw.app.FaceclawAlarms.readLog(context));
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Ambient light sensor access through the custom firmware (CFW image-handler
|
||||
* mode 16, capability token "als16"; see g2flash/patches/als_sensor.c).
|
||||
*
|
||||
* The stock firmware only reads its OPT3001 light sensor while auto-brightness
|
||||
* is on, and then steps the panel brightness itself (visible flicker). The CFW
|
||||
* adds two things:
|
||||
*
|
||||
* - QUERY: one report from the driver's current state.
|
||||
* - PASSIVE polling: the firmware opens and polls the sensor at our interval,
|
||||
* the stock adjuster never touches the panel, and a report arrives when the
|
||||
* reading moved by at least `minDelta` or `heartbeatMs` elapsed. The phone
|
||||
* then applies brightness itself (setBrightness with autoAdjust=false), e.g.
|
||||
* only while the display is blank or during a UI transition.
|
||||
*
|
||||
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
|
||||
*/
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Ask the firmware for one ambient-light report (delivered to listeners). */
|
||||
export function queryAmbientLight(): void {
|
||||
try {
|
||||
activeCommunicator()?.queryAmbientLight();
|
||||
} catch (error) {
|
||||
console.warn(`queryAmbientLight failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start or stop CFW passive light-sensor polling. Starting is idempotent and
|
||||
* re-opens the sensor if the stock firmware closed it, so it is safe to resend
|
||||
* on every session start.
|
||||
*/
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
try {
|
||||
activeCommunicator()?.setAmbientLightPolling(
|
||||
enabled,
|
||||
options.intervalMs ?? 500,
|
||||
options.minDelta ?? 1,
|
||||
options.heartbeatMs ?? 5000,
|
||||
options.bindToLease ?? true,
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
|
||||
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return () => {};
|
||||
const proxy = new com.faceclaw.app.FaceclawAmbientLightListener({
|
||||
onAmbientLight: (body: any) => {
|
||||
const report = decodeAmbientLightReport(toUint8Array(body));
|
||||
if (report) listener(report);
|
||||
},
|
||||
});
|
||||
active.addAmbientLightListener(proxy);
|
||||
return () => {
|
||||
try {
|
||||
activeCommunicator()?.removeAmbientLightListener(proxy);
|
||||
} catch {
|
||||
// The communicator may have been replaced during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
declare const com: any;
|
||||
|
||||
export interface BleTrafficSample {
|
||||
/** Outbound messages (writeFrames calls) since process start. */
|
||||
messages: number;
|
||||
/** Outbound payload bytes (sum of frame sizes) since process start. */
|
||||
bytes: number;
|
||||
/** Display frames whose last message was acked, since process start. */
|
||||
frames: number;
|
||||
}
|
||||
|
||||
/** Running totals of outbound BLE traffic, counted process-wide in FaceclawBleManager. */
|
||||
export function sampleBleTraffic(): BleTrafficSample {
|
||||
const raw = com.faceclaw.app.FaceclawBleManager.sampleOutboundTraffic();
|
||||
return { messages: Number(raw[0]), bytes: Number(raw[1]), frames: Number(raw[2]) };
|
||||
}
|
||||
@@ -9,6 +9,9 @@
|
||||
export type CloudSttTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
/** Optional presentation for Transcribe only; dictation uses plain text. */
|
||||
transcribeText?: string;
|
||||
paragraphBreakAfter?: boolean;
|
||||
};
|
||||
|
||||
export type CloudSttOptions = {
|
||||
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
onReady?: () => void;
|
||||
onDisconnected?: (message: string) => void;
|
||||
};
|
||||
|
||||
export interface CloudSttClient {
|
||||
@@ -25,6 +30,8 @@ export interface CloudSttClient {
|
||||
acceptPcm(pcm: Uint8Array): void;
|
||||
/** End of utterance: produce a final transcript. */
|
||||
finish(): void;
|
||||
/** Commit a pause boundary while keeping the audio session open. */
|
||||
commitSegment?(): void;
|
||||
/** Abandon the session without finalizing. */
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
+34
-6
@@ -5,10 +5,23 @@ export const COMPASS_CHANGED = 15;
|
||||
export const COMPASS_CALIBRATION_STARTED = 16;
|
||||
export const COMPASS_CALIBRATION_COMPLETE = 17;
|
||||
|
||||
export type CompassDiagnostics = {
|
||||
/** 0 uncalibrated .. 3 well calibrated; -1 unavailable. */
|
||||
magneticAccuracy: number;
|
||||
/** 0 none, 1 small/temperature, 2 large; -1 unavailable. */
|
||||
magneticAnomalies: number;
|
||||
/** 0 unknown, 1 GRV (relative), 2 GMRV, 3 RV. */
|
||||
orientationSource: number;
|
||||
flags: number;
|
||||
sampleTimeMs: number;
|
||||
};
|
||||
|
||||
export type CompassEvent = {
|
||||
command: number;
|
||||
/** Magnetic heading in degrees, or -1 for calibration-only events. */
|
||||
headingDegrees: number;
|
||||
/** Absent with stock/older CFW or an unrecognized diagnostic extension. */
|
||||
diagnostics?: CompassDiagnostics;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
@@ -20,22 +33,37 @@ function activeCommunicator(): any {
|
||||
}
|
||||
}
|
||||
|
||||
/** Request CFW mode 10 to start/stop the stock firmware compass. */
|
||||
export function setCompassEnabled(enabled: boolean): void {
|
||||
/**
|
||||
* Request CFW mode 10 to start/stop the stock firmware compass on behalf of
|
||||
* `owner`. The magnetometer is shared: it runs while any owner wants it and
|
||||
* stops once the last one releases it, so apps can't switch each other off.
|
||||
*/
|
||||
export function setCompassEnabled(enabled: boolean, owner = "compass"): void {
|
||||
try {
|
||||
activeCommunicator()?.setCompassEnabled(enabled);
|
||||
activeCommunicator()?.setCompassEnabled(owner, enabled);
|
||||
} catch (error) {
|
||||
console.warn(`setCompassEnabled failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to stock compass heading/calibration notifications. */
|
||||
/**
|
||||
* Subscribe to stock compass heading/calibration notifications. Events are
|
||||
* delivered on the calling thread's Looper, so this works from app workers.
|
||||
*/
|
||||
export function addCompassListener(listener: (event: CompassEvent) => void): () => void {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return () => {};
|
||||
const proxy = new com.faceclaw.app.FaceclawCompassListener({
|
||||
onCompassEvent: (command: number, headingDegrees: number) => {
|
||||
listener({ command: Number(command), headingDegrees: Number(headingDegrees) });
|
||||
onCompassEvent: (command: number, headingDegrees: number, magneticAccuracy: number,
|
||||
magneticAnomalies: number, orientationSource: number, diagnosticFlags: number, sampleTimeMs: number) => {
|
||||
listener({
|
||||
command: Number(command), headingDegrees: Number(headingDegrees),
|
||||
diagnostics: diagnosticFlags >= 0 ? {
|
||||
magneticAccuracy: Number(magneticAccuracy), magneticAnomalies: Number(magneticAnomalies),
|
||||
orientationSource: Number(orientationSource), flags: Number(diagnosticFlags),
|
||||
sampleTimeMs: Number(sampleTimeMs),
|
||||
} : undefined,
|
||||
});
|
||||
},
|
||||
});
|
||||
active.addCompassListener(proxy);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { encodeBase64 } from "./cloud-stt";
|
||||
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
|
||||
const MODEL_ID = "scribe_v2_realtime";
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export type ElevenLabsTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type ElevenLabsSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
|
||||
export type ElevenLabsSttOptions = CloudSttOptions;
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private latestText = "";
|
||||
private readonly commits: boolean[] = [];
|
||||
|
||||
constructor(private readonly options: ElevenLabsSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: flush and commit for a final transcript. */
|
||||
finish(): void {
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
const commitMessage = JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: "",
|
||||
commit: true,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
});
|
||||
if (this.open) {
|
||||
this.trySend(commitMessage);
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) this.sendChunk("__commit__");
|
||||
else this.pendingChunks.push("__commit__");
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
this.trySend(
|
||||
JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: base64OrCommit,
|
||||
commit: false,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}),
|
||||
);
|
||||
const commit = base64OrCommit === "__commit__";
|
||||
this.trySend(JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: commit ? "" : base64OrCommit,
|
||||
commit,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}));
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("ElevenLabs send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("elevenlabs send failed", error);
|
||||
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "committed_transcript":
|
||||
case "committed_transcript_with_timestamps":
|
||||
this.latestText = String(message.text ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
|
||||
this.latestText = "";
|
||||
return;
|
||||
case "committed_transcript_with_timestamps":
|
||||
// The timestamp event follows the plain commit; do not append twice.
|
||||
return;
|
||||
case "rate_limited":
|
||||
this.options.onDisconnected?.("ElevenLabs rate limited.");
|
||||
return;
|
||||
case "error":
|
||||
case "auth_error":
|
||||
case "quota_exceeded":
|
||||
case "rate_limited":
|
||||
case "input_error":
|
||||
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
|
||||
return;
|
||||
|
||||
@@ -22,6 +22,8 @@ export type CommunicatorState = {
|
||||
export type HeadsetBatteryState = {
|
||||
battery: number;
|
||||
chargingStatus: number;
|
||||
ringBattery?: number;
|
||||
ringChargingStatus?: number;
|
||||
};
|
||||
|
||||
export type FrameMetrics = {
|
||||
@@ -120,6 +122,11 @@ function nonNegativeNumber(value: number): number {
|
||||
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
|
||||
}
|
||||
|
||||
/** A 0..1 brightness factor as the compositor's 0..256 fixed-point form. */
|
||||
export function dimFactor256(factor: number): number {
|
||||
return Math.round(Math.max(0, Math.min(1, factor)) * 256);
|
||||
}
|
||||
|
||||
export class FaceclawCommunicatorBridge {
|
||||
private readonly communicator: any;
|
||||
private readonly listenerProxy: any;
|
||||
@@ -184,10 +191,12 @@ export class FaceclawCommunicatorBridge {
|
||||
frameTimings.logFrame(event.frameId, "input event received on JS side");
|
||||
this.emitAsync(this.ringListeners, event);
|
||||
},
|
||||
onBatteryState: (headsetBattery: number, headsetCharging: number) => {
|
||||
onBatteryState: (headsetBattery: number, headsetCharging: number, ringBattery: number, ringCharging: number) => {
|
||||
const state = {
|
||||
battery: Number(headsetBattery),
|
||||
chargingStatus: Number(headsetCharging),
|
||||
ringBattery: Number(ringBattery),
|
||||
ringChargingStatus: Number(ringCharging),
|
||||
};
|
||||
this.emitAsync(this.batteryListeners, state);
|
||||
},
|
||||
@@ -524,6 +533,18 @@ export class FaceclawCommunicatorBridge {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Dim every compositor surface whose zOrder is below `belowZOrder` to
|
||||
* `factor` (0..1; 1 = no dimming) of its brightness; takes effect at the
|
||||
* next composite. How a shell overlay's Layer.dimUnderneath reaches the
|
||||
* window surfaces beneath the shell surface.
|
||||
*/
|
||||
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
|
||||
await this.enqueueJavaCall(() => {
|
||||
this.communicator.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
|
||||
});
|
||||
}
|
||||
|
||||
/** Show or hide a compositor surface; takes effect at the next composite. */
|
||||
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
|
||||
await this.enqueueJavaCall(() => {
|
||||
|
||||
@@ -6,20 +6,32 @@ export type FlashPromptState =
|
||||
| "connecting"
|
||||
| "connected"
|
||||
| "prompting"
|
||||
| "battery"
|
||||
| "result"
|
||||
| "cancelled"
|
||||
| "timeout"
|
||||
| "disconnected"
|
||||
| "error";
|
||||
|
||||
/** Per-arm battery percent; null when that arm didn't answer. */
|
||||
export type FlashPromptBattery = { right: number | null; left: number | null };
|
||||
|
||||
function normalizePercent(value: number): number | null {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) && n >= 0 ? n : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* TS wrapper around the native FaceclawFlashPromptCommunicator — the
|
||||
* stock-firmware-compatible BLE path that shows the pre-flash confirmation on
|
||||
* the glasses and reports the user's Yes/No choice. Separate from
|
||||
* FaceclawCommunicatorBridge on purpose (different, minimal protocol subset).
|
||||
*
|
||||
* Talks to the right arm only: the lenses relay messages between themselves,
|
||||
* and acks/events always come from the right arm.
|
||||
* Connects and authenticates both arms (so any OS pairing prompts happen up
|
||||
* front, before flashing), shows the prompt via the right arm (the lenses relay
|
||||
* messages between themselves, and acks/events always come from the right
|
||||
* arm), then reads each arm's battery. With `skipPrompt` the on-glasses
|
||||
* confirmation is omitted and only the battery is read.
|
||||
*/
|
||||
export class FlashPromptCommunicator {
|
||||
private readonly communicator: any;
|
||||
@@ -27,20 +39,32 @@ export class FlashPromptCommunicator {
|
||||
private readonly logListeners = new Set<(line: string) => void>();
|
||||
private readonly stateListeners = new Set<(state: FlashPromptState, detail: string) => void>();
|
||||
private readonly resultListeners = new Set<(approved: boolean) => void>();
|
||||
private readonly batteryListeners = new Set<(battery: FlashPromptBattery) => void>();
|
||||
|
||||
constructor(rightAddress: string, warningText: string) {
|
||||
constructor(
|
||||
addresses: { right: string; left: string },
|
||||
warningText: string,
|
||||
options?: { skipPrompt?: boolean },
|
||||
) {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
if (!context) throw new Error("Android application context unavailable");
|
||||
|
||||
this.communicator = new com.faceclaw.app.FaceclawFlashPromptCommunicator(
|
||||
context,
|
||||
rightAddress,
|
||||
addresses.right,
|
||||
addresses.left,
|
||||
warningText,
|
||||
Boolean(options?.skipPrompt),
|
||||
);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawFlashPromptListener({
|
||||
onLog: (line: string) => this.emit(this.logListeners, String(line)),
|
||||
onState: (state: string, detail: string) =>
|
||||
this.emit(this.stateListeners, String(state) as FlashPromptState, String(detail ?? "")),
|
||||
onBattery: (rightPercent: number, leftPercent: number) =>
|
||||
this.emit(this.batteryListeners, {
|
||||
right: normalizePercent(rightPercent),
|
||||
left: normalizePercent(leftPercent),
|
||||
}),
|
||||
onResult: (approved: boolean) => this.emit(this.resultListeners, Boolean(approved)),
|
||||
});
|
||||
this.communicator.setListener(this.listenerProxy);
|
||||
@@ -56,6 +80,11 @@ export class FlashPromptCommunicator {
|
||||
return () => this.stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
onBattery(listener: (battery: FlashPromptBattery) => void): () => void {
|
||||
this.batteryListeners.add(listener);
|
||||
return () => this.batteryListeners.delete(listener);
|
||||
}
|
||||
|
||||
onResult(listener: (approved: boolean) => void): () => void {
|
||||
this.resultListeners.add(listener);
|
||||
return () => this.resultListeners.delete(listener);
|
||||
|
||||
@@ -84,6 +84,7 @@ type TerminalDataKind = "snapshot" | "output";
|
||||
export class G2MirrorClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private connectionGeneration = 0;
|
||||
private phase: G2MirrorPhase = "idle";
|
||||
private status = "Not connected.";
|
||||
private sessions: G2MirrorSession[] = [];
|
||||
@@ -168,14 +169,21 @@ export class G2MirrorClient {
|
||||
return () => this.titleListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Grid dimensions for the next connection's handshake. */
|
||||
setViewport(cols: number, rows: number): void {
|
||||
this.options.cols = cols;
|
||||
this.options.rows = rows;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
this.stopped = false;
|
||||
const generation = ++this.connectionGeneration;
|
||||
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
|
||||
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.stopped) return;
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
this.setState("connecting", "Authenticating...");
|
||||
this.send({
|
||||
type: "init",
|
||||
@@ -187,15 +195,15 @@ export class G2MirrorClient {
|
||||
});
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (this.stopped) return;
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
this.handleMessage(String(message));
|
||||
},
|
||||
onClosed: (code: number, reason: string) => {
|
||||
if (this.stopped) return;
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
this.handleConnectionLost(`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""}).`);
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.stopped) return;
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
this.handleConnectionLost(`Connection failed: ${shortenError(String(message))}`);
|
||||
},
|
||||
});
|
||||
@@ -209,6 +217,7 @@ export class G2MirrorClient {
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
++this.connectionGeneration;
|
||||
this.rejectAllPendingLaunches("client stopped");
|
||||
this.clearListRefreshTimer();
|
||||
if (this.ws) {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Magnetic declination from Android's built-in World Magnetic Model. Declination
|
||||
* is the angle from magnetic north to true north (east positive), so
|
||||
* `true = magnetic + declination`. No network access is involved; the model is
|
||||
* baked into the platform and is accurate to well under a degree, which is far
|
||||
* inside the magnetometer's own noise.
|
||||
*/
|
||||
declare const android: any;
|
||||
declare const global: any;
|
||||
|
||||
/** Declination in degrees at a place and time, or null off-Android. */
|
||||
export function magneticDeclinationDegrees(
|
||||
latitude: number,
|
||||
longitude: number,
|
||||
timestampMs: number = Date.now(),
|
||||
altitudeMeters = 0,
|
||||
): number | null {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
const field = new android.hardware.GeomagneticField(latitude, longitude, altitudeMeters, timestampMs);
|
||||
const declination = Number(field.getDeclination());
|
||||
return Number.isFinite(declination) ? declination : null;
|
||||
} catch (error) {
|
||||
console.warn(`GeomagneticField failed: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,39 @@ export function isDecodableImageFile(name: string): boolean {
|
||||
return DECODABLE_IMAGE.test(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* How source gray maps onto the display's 16 levels when converting to a
|
||||
* GrayImage. The default (gamma 1, no dither) is a flat conversion with
|
||||
* per-pixel rounding, right for UI raster such as icons and map tiles.
|
||||
*/
|
||||
export type GrayImageTone = {
|
||||
/**
|
||||
* Tone curve out = in^gamma (in 0..1). gamma > 1 darkens midtones: source
|
||||
* pixels are sRGB-encoded but the G2 drives its levels roughly linearly,
|
||||
* so untouched photos read washed out.
|
||||
*/
|
||||
gamma?: number;
|
||||
/**
|
||||
* Floyd-Steinberg error diffusion onto the 16 display levels instead of
|
||||
* rounding each pixel independently, which bands smooth gradients.
|
||||
*/
|
||||
dither?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Gamma for continuous-tone images (photos, album art): 2.2 undoes the sRGB
|
||||
* encoding outright. Lower it toward ~1.8 if shadows crush.
|
||||
*/
|
||||
export const PHOTO_GAMMA = 2.2;
|
||||
|
||||
/** Tone preset for continuous-tone images: sRGB gamma plus dithering. */
|
||||
export const PHOTO_TONE: Readonly<Required<GrayImageTone>> = { gamma: PHOTO_GAMMA, dither: true };
|
||||
|
||||
/** Resolve a tone to the (gamma, dither) pair the Java helpers take. */
|
||||
export function resolveGrayImageTone(tone?: GrayImageTone): { gamma: number; dither: boolean } {
|
||||
return { gamma: tone?.gamma ?? 1, dither: tone?.dither ?? false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a gray packet ([widthLo, widthHi, heightLo, heightHi, pixels...],
|
||||
* the format the Java helpers return) into a GrayImage; null when empty or
|
||||
@@ -34,12 +67,20 @@ export function grayImageFromPacket(raw: ArrayLike<number> | null | undefined):
|
||||
/**
|
||||
* Load an image file as grayscale, downscaled to fit maxWidth x maxHeight
|
||||
* (aspect preserved, never upscaled); null when unreadable or undecodable.
|
||||
* Pass PHOTO_TONE for continuous-tone images; the default is a flat
|
||||
* conversion for UI raster.
|
||||
*/
|
||||
export function loadImageFileAsGray(path: string, maxWidth: number, maxHeight: number): GrayImage | null {
|
||||
export function loadImageFileAsGray(
|
||||
path: string,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
tone?: GrayImageTone,
|
||||
): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
const { gamma, dither } = resolveGrayImageTone(tone);
|
||||
return grayImageFromPacket(
|
||||
com.faceclaw.app.ImageFileLoader.loadGray(path, Math.round(maxWidth), Math.round(maxHeight)),
|
||||
com.faceclaw.app.ImageFileLoader.loadGray(path, Math.round(maxWidth), Math.round(maxHeight), gamma, dither),
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(`loadImageFileAsGray failed for ${path}: ${error}`);
|
||||
|
||||
+49
-9
@@ -18,6 +18,23 @@ const DIRECTIONS_ROOT = "https://api.mapbox.com/directions/v5";
|
||||
const STATIC_ROOT = "https://api.mapbox.com/styles/v1";
|
||||
/** Classic dark style; readable after gray conversion. Swappable for a custom Studio style later. */
|
||||
const MAP_STYLE = "mapbox/dark-v11";
|
||||
/**
|
||||
* Render-time style override: a black background inserted beneath the road
|
||||
* stack so land, landuse, water, and building fills all come out black on the
|
||||
* lens (dark-v11's own fills are dark gray, which the lens shows as a lit
|
||||
* haze). Static Images allows one added layer per request; `before_layer`
|
||||
* must name a layer in the style, so candidates are tried in order and a
|
||||
* rejected one is skipped for the rest of the session, ending with the plain
|
||||
* style (the worker's levelling pass then does what it can client-side).
|
||||
*/
|
||||
const STYLE_OVERRIDE_LAYER = JSON.stringify({
|
||||
id: "faceclaw-blackout",
|
||||
type: "background",
|
||||
paint: { "background-color": "#000000" },
|
||||
});
|
||||
/** First road layer of dark-v11, then a layer common to every Streets-based style. */
|
||||
const STYLE_OVERRIDE_BEFORE_LAYERS = ["tunnel-path-trail", "building"];
|
||||
let styleOverrideIndex = 0;
|
||||
const FETCH_TIMEOUT_MS = 15_000;
|
||||
|
||||
export type RouteProfile = "driving" | "walking" | "cycling";
|
||||
@@ -188,18 +205,32 @@ export async function fetchStaticMapGray(options: {
|
||||
: `${round6(options.camera.longitude)},${round6(options.camera.latitude)},` +
|
||||
`${options.camera.zoom.toFixed(2)},${Math.round(normalizeBearing(options.camera.bearingDeg))}`;
|
||||
const size = `${Math.round(options.width)}x${Math.round(options.height)}`;
|
||||
const params = queryString({
|
||||
access_token: requireToken(),
|
||||
logo: "false",
|
||||
attribution: "false",
|
||||
});
|
||||
const url = `${STATIC_ROOT}/${MAP_STYLE}/static/${overlay}${camera}/${size}?${params}`;
|
||||
let bytes: ArrayBuffer;
|
||||
for (;;) {
|
||||
const beforeLayer = STYLE_OVERRIDE_BEFORE_LAYERS[styleOverrideIndex];
|
||||
const params: Record<string, string> = {
|
||||
access_token: requireToken(),
|
||||
logo: "false",
|
||||
attribution: "false",
|
||||
};
|
||||
if (beforeLayer !== undefined) {
|
||||
params.addlayer = STYLE_OVERRIDE_LAYER;
|
||||
params.before_layer = beforeLayer;
|
||||
}
|
||||
const url = `${STATIC_ROOT}/${MAP_STYLE}/static/${overlay}${camera}/${size}?${queryString(params)}`;
|
||||
|
||||
const response = await fetchWithTimeout(url, "Mapbox static map");
|
||||
if (!response.ok) {
|
||||
const response = await fetchWithTimeout(url, "Mapbox static map");
|
||||
if (response.ok) {
|
||||
bytes = await response.arrayBuffer();
|
||||
break;
|
||||
}
|
||||
if (beforeLayer !== undefined && isStyleOverrideRejection(response.status)) {
|
||||
console.warn(`Mapbox rejected the map style override (before_layer=${beforeLayer}, HTTP ${response.status}); falling back`);
|
||||
styleOverrideIndex++;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Mapbox static map failed (HTTP ${response.status}).`);
|
||||
}
|
||||
const bytes = await response.arrayBuffer();
|
||||
if (!global.isAndroid) throw new Error("Map decoding is only available on Android.");
|
||||
const image = grayImageFromPacket(
|
||||
com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(bytes, Math.round(options.width), Math.round(options.height)),
|
||||
@@ -281,6 +312,15 @@ async function fetchWithTimeout(url: string, label: string): Promise<Response> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A client error that plausibly blames the added layer (an unknown
|
||||
* before_layer, or a style without it), as opposed to token or quota trouble
|
||||
* that the plain request would hit just the same.
|
||||
*/
|
||||
function isStyleOverrideRejection(status: number): boolean {
|
||||
return status >= 400 && status < 500 && status !== 401 && status !== 403 && status !== 429;
|
||||
}
|
||||
|
||||
function normalizeBearing(bearingDeg: number): number {
|
||||
return ((bearingDeg % 360) + 360) % 360;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { getStringSetting, setStringSetting } from "./settings-store";
|
||||
|
||||
export const MEDIA_APPS_KEY = "music.apps";
|
||||
|
||||
// Exact package IDs only: browsers, social feeds and messaging apps commonly
|
||||
// publish sessions for embedded clips. Users can explicitly enable any of them.
|
||||
export const DEFAULT_IGNORED_MEDIA_APPS: readonly string[] = [
|
||||
"com.android.chrome",
|
||||
"org.mozilla.firefox",
|
||||
"com.microsoft.emmx",
|
||||
"com.sec.android.app.sbrowser",
|
||||
"com.facebook.katana",
|
||||
"com.facebook.orca",
|
||||
"com.instagram.android",
|
||||
"com.whatsapp",
|
||||
"org.telegram.messenger",
|
||||
"com.snapchat.android",
|
||||
"com.reddit.frontpage",
|
||||
"com.twitter.android",
|
||||
"com.discord",
|
||||
];
|
||||
|
||||
export type MediaApp = {
|
||||
packageName: string;
|
||||
appName: string;
|
||||
/** Absent until the user chooses; built-in defaults can evolve independently. */
|
||||
enabled?: boolean;
|
||||
};
|
||||
|
||||
/** Keep discovered apps after their sessions end, including ignored apps. */
|
||||
export function readMediaApps(): MediaApp[] {
|
||||
try {
|
||||
const parsed = JSON.parse(getStringSetting(MEDIA_APPS_KEY, "[]"));
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
const apps = new Map<string, MediaApp>();
|
||||
for (const app of parsed) {
|
||||
if (!app || typeof app.packageName !== "string" || !app.packageName) continue;
|
||||
apps.set(app.packageName, {
|
||||
packageName: app.packageName,
|
||||
appName: typeof app.appName === "string" && app.appName ? app.appName : app.packageName,
|
||||
...(typeof app.enabled === "boolean" ? { enabled: app.enabled } : {}),
|
||||
});
|
||||
}
|
||||
return Array.from(apps.values());
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function rememberMediaApps(discovered: { packageName: string; appName: string }[]): void {
|
||||
const apps = new Map(readMediaApps().map((app) => [app.packageName, app]));
|
||||
let changed = false;
|
||||
for (const app of discovered) {
|
||||
if (!app.packageName) continue;
|
||||
const previous = apps.get(app.packageName);
|
||||
const appName = app.appName || previous?.appName || app.packageName;
|
||||
if (!previous || previous.appName !== appName) {
|
||||
apps.set(app.packageName, { ...previous, packageName: app.packageName, appName });
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) setStringSetting(MEDIA_APPS_KEY, JSON.stringify(Array.from(apps.values())));
|
||||
}
|
||||
|
||||
export function ignoredMediaPackages(): string[] {
|
||||
const ignored = new Set(DEFAULT_IGNORED_MEDIA_APPS);
|
||||
for (const app of readMediaApps()) {
|
||||
if (app.enabled === true) ignored.delete(app.packageName);
|
||||
else if (app.enabled === false) ignored.add(app.packageName);
|
||||
}
|
||||
return Array.from(ignored).sort();
|
||||
}
|
||||
|
||||
export function isMediaAppEnabled(packageName: string): boolean {
|
||||
return !ignoredMediaPackages().includes(packageName);
|
||||
}
|
||||
|
||||
export function setMediaAppEnabled(app: MediaApp, enabled: boolean): void {
|
||||
if (!app.packageName) return;
|
||||
const apps = new Map(readMediaApps().map((item) => [item.packageName, item]));
|
||||
apps.set(app.packageName, { packageName: app.packageName, appName: app.appName || app.packageName, enabled });
|
||||
setStringSetting(MEDIA_APPS_KEY, JSON.stringify(Array.from(apps.values())));
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -34,11 +36,12 @@ export class FaceclawMediaBrowserBridge {
|
||||
private readonly pendingConnects = new Map<number, PendingRequest<string>>();
|
||||
private readonly pendingBrowses = new Map<number, PendingRequest<MediaBrowseItem[]>>();
|
||||
private appsCache: MediaBrowserApp[] | null = null;
|
||||
private connectedPackage = "";
|
||||
|
||||
/** Installed apps exposing a media browser service; cached after the first call. */
|
||||
listBrowsableApps(refresh = false): MediaBrowserApp[] {
|
||||
if (!global.isAndroid) return [];
|
||||
if (this.appsCache && !refresh) return this.appsCache;
|
||||
if (this.appsCache && !refresh) return this.enabledApps();
|
||||
this.ensureBrowser();
|
||||
if (!this.browser) return [];
|
||||
try {
|
||||
@@ -52,11 +55,17 @@ export class FaceclawMediaBrowserBridge {
|
||||
serviceClass: String(app.serviceClass ?? ""),
|
||||
appName: String(app.appName ?? ""),
|
||||
}));
|
||||
rememberMediaApps(this.appsCache);
|
||||
} catch (error) {
|
||||
console.warn(`media browser app list parse failed: ${error}`);
|
||||
this.appsCache = [];
|
||||
}
|
||||
return this.appsCache;
|
||||
return this.enabledApps();
|
||||
}
|
||||
|
||||
private enabledApps(): MediaBrowserApp[] {
|
||||
const ignored = new Set(ignoredMediaPackages());
|
||||
return (this.appsCache ?? []).filter((app) => !ignored.has(app.packageName));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -65,9 +74,11 @@ export class FaceclawMediaBrowserBridge {
|
||||
*/
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
if (!global.isAndroid) return Promise.reject(new Error("Android only"));
|
||||
if (!isMediaAppEnabled(app.packageName)) return Promise.reject(new Error("App is ignored in Music settings"));
|
||||
this.ensureBrowser();
|
||||
if (!this.browser) return Promise.reject(new Error("Media browser unavailable"));
|
||||
const requestId = this.nextRequestId++;
|
||||
this.connectedPackage = app.packageName;
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
this.pendingConnects.set(requestId, { resolve, reject });
|
||||
this.browser.connect(requestId, app.packageName, app.serviceClass);
|
||||
@@ -77,6 +88,10 @@ export class FaceclawMediaBrowserBridge {
|
||||
/** Fetch one level of the connected service's content tree. */
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
if (!global.isAndroid) return Promise.reject(new Error("Android only"));
|
||||
if (this.connectedPackage && !isMediaAppEnabled(this.connectedPackage)) {
|
||||
this.disconnect();
|
||||
return Promise.reject(new Error("App is ignored in Music settings"));
|
||||
}
|
||||
this.ensureBrowser();
|
||||
if (!this.browser) return Promise.reject(new Error("Media browser unavailable"));
|
||||
const requestId = this.nextRequestId++;
|
||||
@@ -89,6 +104,7 @@ export class FaceclawMediaBrowserBridge {
|
||||
/** Start playback of a browsed item through the connected player's session. */
|
||||
playFromMediaId(mediaId: string): void {
|
||||
if (!global.isAndroid) return;
|
||||
if (!this.connectedPackage || !isMediaAppEnabled(this.connectedPackage)) return;
|
||||
this.browser?.playFromMediaId(mediaId);
|
||||
}
|
||||
|
||||
@@ -96,6 +112,7 @@ export class FaceclawMediaBrowserBridge {
|
||||
disconnect(): void {
|
||||
if (!global.isAndroid) return;
|
||||
this.rejectAllPending("Disconnected");
|
||||
this.connectedPackage = "";
|
||||
this.browser?.disconnect();
|
||||
}
|
||||
|
||||
@@ -115,6 +132,11 @@ export class FaceclawMediaBrowserBridge {
|
||||
throw new Error("Android application context unavailable");
|
||||
}
|
||||
this.browser = new com.faceclaw.app.FaceclawMediaBrowser(context);
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key === MEDIA_APPS_KEY && this.connectedPackage && !isMediaAppEnabled(this.connectedPackage)) {
|
||||
this.disconnect();
|
||||
}
|
||||
});
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawMediaBrowserListener({
|
||||
onConnectResult: (requestId: number, connected: boolean, rootId: string, error: string) => {
|
||||
const pending = this.pendingConnects.get(Number(requestId));
|
||||
|
||||
@@ -1,17 +1,11 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { grayImageFromPacket } from "./image-files";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { ignoredMediaPackages, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Tone curve applied to album art before it is dithered down to the display's
|
||||
* 16 gray levels (out = in^gamma, in 0..1). Cover art is sRGB-encoded but the
|
||||
* G2 drives its levels roughly linearly, so untouched art reads washed out;
|
||||
* 2.2 undoes the sRGB encoding. Lower it toward ~1.8 if shadows crush.
|
||||
*/
|
||||
const ALBUM_ART_GAMMA = 2.2;
|
||||
|
||||
export type MediaQueueItem = {
|
||||
id: number;
|
||||
title: string;
|
||||
@@ -141,16 +135,16 @@ export class FaceclawMediaControllerBridge {
|
||||
}
|
||||
|
||||
/**
|
||||
* Album art for the current item, grayscale (gamma-corrected and dithered
|
||||
* onto the display's 16 levels), scaled to fit maxSize; null when the
|
||||
* player provides none.
|
||||
* Album art for the current item, grayscale with the shared photo tone
|
||||
* (gamma-corrected and dithered onto the display's 16 levels), scaled to
|
||||
* fit maxSize; null when the player provides none.
|
||||
*/
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
if (!global.isAndroid) return null;
|
||||
this.ensureController();
|
||||
if (!this.controller) return null;
|
||||
return grayImageFromPacket(
|
||||
this.controller.getAlbumArtGray(Math.round(maxSize), ALBUM_ART_GAMMA),
|
||||
this.controller.getAlbumArtGray(Math.round(maxSize), PHOTO_TONE.gamma, PHOTO_TONE.dither),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -181,7 +175,22 @@ export class FaceclawMediaControllerBridge {
|
||||
throw new Error("Android application context unavailable");
|
||||
}
|
||||
this.controller = new com.faceclaw.app.FaceclawMediaController(context);
|
||||
// Install defaults and saved overrides before starting session discovery.
|
||||
this.controller.setIgnoredPackagesJson(JSON.stringify(ignoredMediaPackages()));
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key === MEDIA_APPS_KEY) {
|
||||
this.controller.setIgnoredPackagesJson(JSON.stringify(ignoredMediaPackages()));
|
||||
}
|
||||
});
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawMediaControllerListener({
|
||||
onSessionAppsChanged: (appsJson: string) => {
|
||||
try {
|
||||
const apps = JSON.parse(String(appsJson));
|
||||
rememberMediaApps(apps);
|
||||
} catch (error) {
|
||||
console.warn(`media session apps parse failed: ${error}`);
|
||||
}
|
||||
},
|
||||
onStateChange: (
|
||||
playbackState: string,
|
||||
packageName: string,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { loadNightscoutSettings, nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings";
|
||||
import { nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings";
|
||||
import { fetchWithUserAgent } from "../util/http";
|
||||
|
||||
export type NightscoutPoint = {
|
||||
@@ -33,6 +33,10 @@ export type NightscoutState = {
|
||||
iob: number | null;
|
||||
cob: number | null;
|
||||
cageTimestampMs: number | null;
|
||||
reservoirUnits: number | null;
|
||||
batteryVoltage: number | null;
|
||||
loopTimestampMs: number | null;
|
||||
/** Pump communication age and operating state; measurements are separate. */
|
||||
pumpStatus: string;
|
||||
openapsStatusShort: string;
|
||||
carbs: NightscoutCarbEvent[];
|
||||
@@ -121,6 +125,9 @@ const DEFAULT_NIGHTSCOUT_STATE: NightscoutState = {
|
||||
iob: null,
|
||||
cob: null,
|
||||
cageTimestampMs: null,
|
||||
reservoirUnits: null,
|
||||
batteryVoltage: null,
|
||||
loopTimestampMs: null,
|
||||
pumpStatus: "",
|
||||
openapsStatusShort: "--",
|
||||
carbs: [],
|
||||
@@ -259,6 +266,9 @@ export class NightscoutBridge {
|
||||
iob: Number.isFinite(iob) ? iob : null,
|
||||
cob,
|
||||
cageTimestampMs: latestSiteChange > 0 ? latestSiteChange : null,
|
||||
reservoirUnits: firstFiniteNumber(latestDeviceStatus?.pump?.reservoir),
|
||||
batteryVoltage: firstFiniteNumber(latestDeviceStatus?.pump?.battery?.voltage),
|
||||
loopTimestampMs: loopTimestampMs > 0 ? loopTimestampMs : null,
|
||||
pumpStatus,
|
||||
openapsStatusShort: loopTimestampMs > 0 ? formatElapsedShort(nowMs - loopTimestampMs) : "--",
|
||||
carbs: parsedTreatments.carbs,
|
||||
@@ -359,6 +369,8 @@ function parseTimestampMs(value: unknown): number {
|
||||
|
||||
function firstFiniteNumber(...values: unknown[]): number | null {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "number" && typeof value !== "string") continue;
|
||||
if (typeof value === "string" && !value.trim()) continue;
|
||||
const numeric = Number(value);
|
||||
if (Number.isFinite(numeric)) {
|
||||
return numeric;
|
||||
@@ -371,8 +383,6 @@ function formatPumpStatus(status: NightscoutDeviceStatusResponse | undefined, no
|
||||
if (!status?.pump) {
|
||||
return "--";
|
||||
}
|
||||
const reservoir = firstFiniteNumber(status.pump.reservoir);
|
||||
const voltage = firstFiniteNumber(status.pump.battery?.voltage);
|
||||
const commTimestampMs = Math.max(
|
||||
parseTimestampMs(status.pump.status?.timestamp),
|
||||
parseTimestampMs(status.pump.clock),
|
||||
@@ -380,12 +390,6 @@ function formatPumpStatus(status: NightscoutDeviceStatusResponse | undefined, no
|
||||
);
|
||||
const stateLabel = summarizePumpState(status);
|
||||
const parts: string[] = [];
|
||||
if (reservoir !== null) {
|
||||
parts.push(`${Math.round(reservoir)}U`);
|
||||
}
|
||||
if (voltage !== null) {
|
||||
parts.push(`${voltage.toFixed(2)}v`);
|
||||
}
|
||||
if (commTimestampMs > 0) {
|
||||
parts.push(`${formatElapsedShort(nowMs - commTimestampMs)} ago`);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { logCurrent, spanCurrent } from "./frame-timings";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { rememberNotificationSources } from "./notification-sources";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
const ICON_SIZE = 24;
|
||||
/** Ask the native listener for every active source, including those below the list's display limit. */
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff;
|
||||
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
|
||||
// notification is posted or dismissed (see invalidateIconCaches callers), so
|
||||
// the tray is kept fresh by invalidation, not by expiry. A short TTL just
|
||||
@@ -158,7 +161,9 @@ export function readActiveNotifications(maxNotifications = 50): AndroidNotificat
|
||||
);
|
||||
const parsed = JSON.parse(json);
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.map(normalizeNotification).filter((item): item is AndroidNotification => Boolean(item));
|
||||
const notifications = parsed.map(normalizeNotification).filter((item): item is AndroidNotification => Boolean(item));
|
||||
rememberNotificationSources(notifications);
|
||||
return notifications;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { getStringSetting, setStringSetting } from "./settings-store";
|
||||
|
||||
const SOURCES_KEY = "notifications.sources";
|
||||
|
||||
export type NotificationSource = {
|
||||
packageName: string;
|
||||
appName: string;
|
||||
showOnGlasses: boolean;
|
||||
};
|
||||
|
||||
/** Sources outlive their active notifications so a muted app can always be re-enabled. */
|
||||
export function readNotificationSources(): NotificationSource[] {
|
||||
try {
|
||||
const parsed = JSON.parse(getStringSetting(SOURCES_KEY, "[]"));
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((source) => source && typeof source.packageName === "string" && source.packageName)
|
||||
.map((source) => ({
|
||||
packageName: source.packageName,
|
||||
appName: typeof source.appName === "string" && source.appName ? source.appName : source.packageName,
|
||||
showOnGlasses: source.showOnGlasses !== false,
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function rememberNotificationSources(notifications: { packageName: string; appName: string }[]): void {
|
||||
const sources = new Map(readNotificationSources().map((source) => [source.packageName, source]));
|
||||
let changed = false;
|
||||
for (const notification of notifications) {
|
||||
if (!notification.packageName) continue;
|
||||
const previous = sources.get(notification.packageName);
|
||||
const appName = notification.appName || previous?.appName || notification.packageName;
|
||||
if (!previous || previous.appName !== appName) {
|
||||
sources.set(notification.packageName, {
|
||||
packageName: notification.packageName,
|
||||
appName,
|
||||
showOnGlasses: previous?.showOnGlasses ?? true,
|
||||
});
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
|
||||
}
|
||||
|
||||
export function shouldShowNotificationOnGlasses(packageName: string): boolean {
|
||||
return readNotificationSources().find((source) => source.packageName === packageName)?.showOnGlasses ?? true;
|
||||
}
|
||||
|
||||
export function setNotificationSourceEnabled(source: { packageName: string; appName: string }, enabled: boolean): void {
|
||||
if (!source.packageName) return;
|
||||
const sources = new Map(readNotificationSources().map((item) => [item.packageName, item]));
|
||||
sources.set(source.packageName, {
|
||||
packageName: source.packageName,
|
||||
appName: source.appName || source.packageName,
|
||||
showOnGlasses: enabled,
|
||||
});
|
||||
setStringSetting(SOURCES_KEY, JSON.stringify(Array.from(sources.values())));
|
||||
}
|
||||
+74
-16
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private readonly upsampler = new PcmUpsampler();
|
||||
private latestText = "";
|
||||
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
|
||||
private readonly commits: boolean[] = [];
|
||||
private committed = false;
|
||||
|
||||
constructor(private readonly options: OpenAiSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`OpenAI connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
);
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: commit the buffer for a final transcript. */
|
||||
finish(): void {
|
||||
this.committed = true;
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) {
|
||||
this.sendChunk("__commit__");
|
||||
} else {
|
||||
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.committed = true;
|
||||
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
||||
return;
|
||||
}
|
||||
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("OpenAI send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("openai send failed", error);
|
||||
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
case "session.updated":
|
||||
this.options.onStatus("Listening (OpenAI)...");
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
this.latestText += String(message.delta ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
case "input_audio_buffer.committed": {
|
||||
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
this.latestText = String(message.transcript ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
// The utterance is done; nothing further is coming after the commit.
|
||||
if (this.committed) this.stop();
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.delta": {
|
||||
this.item(String(message.item_id)).text += String(message.delta ?? "");
|
||||
this.emitItems();
|
||||
return;
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.completed": {
|
||||
const item = this.item(String(message.item_id));
|
||||
item.text = String(message.transcript ?? item.text);
|
||||
item.final = true;
|
||||
this.emitItems();
|
||||
if (this.committed && !this.items.size && !this.commits.length) this.stop();
|
||||
return;
|
||||
}
|
||||
case "error":
|
||||
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
|
||||
let item = this.items.get(id);
|
||||
if (!item) {
|
||||
item = { text: "", final: false, paragraphBreakAfter: false };
|
||||
this.items.set(id, item);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
private emitItems(): void {
|
||||
// Committed notifications precede transcription events, so Map insertion
|
||||
// order is audio order even when completion events arrive out of order.
|
||||
for (const [id, item] of this.items) {
|
||||
if (!item.final) break;
|
||||
this.items.delete(id);
|
||||
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
|
||||
}
|
||||
if (this.items.size) {
|
||||
const items = [...this.items.values()];
|
||||
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
|
||||
const transcribeText = items.map((item, index) =>
|
||||
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
|
||||
).join("");
|
||||
this.options.onTranscript({ text, transcribeText, isFinal: false });
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Application, isAndroid } from "@nativescript/core";
|
||||
import { getStringSetting, onSettingsStoreChanged } from "./settings-store";
|
||||
|
||||
// Shared with phoneRotationSetting; keep startup independent of the glasses UI.
|
||||
const ROTATION_KEY = "phone.rotation";
|
||||
|
||||
/** Installed once on the UI thread; also covers navigation and activity recreation. */
|
||||
export function registerPhoneRotation(): void {
|
||||
if (!isAndroid) return;
|
||||
|
||||
const apply = (activity = Application.android.foregroundActivity ?? Application.android.startActivity): void => {
|
||||
if (!activity) return;
|
||||
const orientations = android.content.pm.ActivityInfo;
|
||||
const requested = {
|
||||
auto: orientations.SCREEN_ORIENTATION_UNSPECIFIED,
|
||||
portrait: orientations.SCREEN_ORIENTATION_PORTRAIT,
|
||||
landscape: orientations.SCREEN_ORIENTATION_SENSOR_LANDSCAPE,
|
||||
}[getStringSetting(ROTATION_KEY, "auto")] ?? orientations.SCREEN_ORIENTATION_UNSPECIFIED;
|
||||
if (activity.getRequestedOrientation() !== requested) {
|
||||
activity.setRequestedOrientation(requested);
|
||||
}
|
||||
};
|
||||
|
||||
Application.android.on(Application.android.activityCreatedEvent, (args) => apply(args.activity));
|
||||
Application.android.on(Application.android.activityResumedEvent, (args) => apply(args.activity));
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key === ROTATION_KEY) apply();
|
||||
});
|
||||
apply();
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import type { FaceclawCommunicatorBridge, SurfaceOptions } from "./faceclaw-communicator";
|
||||
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
@@ -19,6 +19,7 @@ export type DisplayTarget = Pick<
|
||||
| "configureSurface"
|
||||
| "removeSurface"
|
||||
| "setSurfaceVisible"
|
||||
| "setUnderlayDim"
|
||||
| "setScreenBlanked"
|
||||
| "submitSurfaceFrame"
|
||||
| "waitForFrameFinished"
|
||||
@@ -96,6 +97,10 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
this.compositor.setSurfaceVisible(id, Boolean(visible));
|
||||
}
|
||||
|
||||
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
|
||||
this.compositor.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
|
||||
}
|
||||
|
||||
async setScreenBlanked(blanked: boolean): Promise<void> {
|
||||
this.compositor.setScreenBlanked(Boolean(blanked));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
|
||||
// Ten seconds of unsent PCM. Never replay audio already sent: that would
|
||||
// duplicate words already shown. An extended outage drops the oldest audio.
|
||||
const MAX_PENDING_BYTES = 16000 * 2 * 10;
|
||||
|
||||
/** Owns retries and audio buffering independently of the provider protocol. */
|
||||
export class ReconnectingSttClient implements CloudSttClient {
|
||||
private client: CloudSttClient | null = null;
|
||||
private generation = 0;
|
||||
private stopped = false;
|
||||
private finishing = false;
|
||||
private ready = false;
|
||||
private retry: ReturnType<typeof setTimeout> | null = null;
|
||||
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private attempt = 0;
|
||||
private pending: Uint8Array[] = [];
|
||||
private pendingBytes = 0;
|
||||
private latest: CloudSttTranscriptEvent | null = null;
|
||||
private readonly pause = new SpeechPauseDetector();
|
||||
|
||||
constructor(
|
||||
private readonly create: (options: CloudSttOptions) => CloudSttClient,
|
||||
private readonly options: CloudSttOptions,
|
||||
private readonly continuous: () => boolean,
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.stopped || this.client || this.retry !== null) return;
|
||||
const generation = ++this.generation;
|
||||
const current = () => !this.stopped && generation === this.generation;
|
||||
this.client = this.create({
|
||||
...this.options,
|
||||
onReady: () => {
|
||||
if (!current()) return;
|
||||
this.ready = true;
|
||||
const chunks = this.pending;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
for (const chunk of chunks) this.acceptPcm(chunk);
|
||||
if (this.finishing) this.client?.finish();
|
||||
},
|
||||
onTranscript: (event) => {
|
||||
if (!current()) return;
|
||||
// A handshake alone can immediately be followed by another failure.
|
||||
this.attempt = 0;
|
||||
this.latest = event.isFinal ? null : event;
|
||||
this.options.onTranscript(event);
|
||||
},
|
||||
onStatus: (status) => { if (current()) this.options.onStatus(status); },
|
||||
onError: (message) => {
|
||||
if (!current()) return;
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
},
|
||||
onDisconnected: (message) => {
|
||||
if (current()) this.disconnected(message);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.client.start();
|
||||
} catch (error) {
|
||||
this.disconnected(`Transcription connection failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.stopped || !pcm.length) return;
|
||||
if (!this.ready) {
|
||||
const copy = pcm.slice(-MAX_PENDING_BYTES);
|
||||
this.pending.push(copy);
|
||||
this.pendingBytes += copy.length;
|
||||
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
|
||||
return;
|
||||
}
|
||||
this.client?.acceptPcm(pcm);
|
||||
// A rejected send synchronously reports disconnection. Keep that packet
|
||||
// with the unsent audio instead of losing it at the outage boundary.
|
||||
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
|
||||
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.stopped || this.finishing) return;
|
||||
this.finishing = true;
|
||||
// Releasing the last mic holder must not leave a reconnect loop alive.
|
||||
if (this.retry !== null) {
|
||||
this.preservePartial();
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.finishTimeout = setTimeout(() => this.stop(), 5000);
|
||||
if (this.ready) this.client?.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
++this.generation;
|
||||
if (this.retry !== null) clearTimeout(this.retry);
|
||||
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
|
||||
this.retry = this.finishTimeout = null;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.ready = false;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private preservePartial(): void {
|
||||
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private disconnected(message: string): void {
|
||||
if (this.stopped) return;
|
||||
++this.generation; // Ignore late finals / closes from the abandoned socket.
|
||||
this.ready = false;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.pause.reset();
|
||||
this.preservePartial();
|
||||
if (this.stopped) return; // A transcript listener can release capture.
|
||||
if (/\b(401|402|403)\b/.test(message)) {
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
return;
|
||||
}
|
||||
if (this.finishing) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
|
||||
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
|
||||
this.retry = setTimeout(() => {
|
||||
this.retry = null;
|
||||
this.start();
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
+48
-13
@@ -1,4 +1,5 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
import { TimedTranscript } from "./transcript-format";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
private readonly pendingPcm: Uint8Array[] = [];
|
||||
private pendingFinish = false;
|
||||
private finishing = false;
|
||||
// Concatenation of all final tokens so far.
|
||||
private finalText = "";
|
||||
private formatted = new TimedTranscript();
|
||||
|
||||
constructor(private readonly options: SonioxSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
this.trySendText(
|
||||
JSON.stringify({
|
||||
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
audio_format: "pcm_s16le",
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingPcm.splice(0)) {
|
||||
this.sendPcm(chunk);
|
||||
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.finish();
|
||||
}
|
||||
this.options.onStatus("Listening (Soniox)...");
|
||||
this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`Soniox connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
/** End of utterance: an empty text frame tells Soniox the audio is done. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
this.finishing = true;
|
||||
if (this.open) {
|
||||
this.trySendText("");
|
||||
} else {
|
||||
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingPcm.length = 0;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
this.ws?.sendBinary(toJavaBytes(pcm));
|
||||
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
|
||||
this.options.onDisconnected?.("Soniox audio send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private trySendText(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("Soniox send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
return;
|
||||
}
|
||||
if (message?.error_code != null) {
|
||||
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
|
||||
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
|
||||
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
|
||||
else this.options.onError(error);
|
||||
return;
|
||||
}
|
||||
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
|
||||
let nonFinal = "";
|
||||
let preview: TimedTranscript | null = null;
|
||||
for (const token of tokens) {
|
||||
const tokenText = String(token?.text ?? "");
|
||||
// Markers emitted by endpoint detection / manual finalize; not speech.
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") continue;
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") {
|
||||
if (token?.is_final && this.finalText) {
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
|
||||
this.finalText = "";
|
||||
this.formatted = new TimedTranscript();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (token?.is_final) {
|
||||
this.finalText += tokenText;
|
||||
this.formatted.append({ ...token, text: tokenText });
|
||||
} else {
|
||||
nonFinal += tokenText;
|
||||
preview ??= this.formatted.copy();
|
||||
preview.append({ ...token, text: tokenText });
|
||||
}
|
||||
}
|
||||
if (message?.finished) {
|
||||
this.options.onTranscript({ text: this.finalText, isFinal: true });
|
||||
this.stop();
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
|
||||
if (this.finishing) this.stop();
|
||||
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
|
||||
return;
|
||||
}
|
||||
if (tokens.length > 0) {
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
|
||||
export const TRANSCRIPT_PAUSE_MS = 1500;
|
||||
|
||||
export class SpeechPauseDetector {
|
||||
private noiseFloor = 220;
|
||||
private speaking = false;
|
||||
private quietSamples = 0;
|
||||
|
||||
reset(): void {
|
||||
this.noiseFloor = 220;
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
}
|
||||
|
||||
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
|
||||
accept(pcm: Uint8Array): boolean {
|
||||
const count = pcm.length >> 1;
|
||||
if (!count) return false;
|
||||
let squares = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
|
||||
squares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(squares / count);
|
||||
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
|
||||
if (rms >= threshold) {
|
||||
this.speaking = true;
|
||||
this.quietSamples = 0;
|
||||
} else {
|
||||
// Slowly follow background levels without learning speech as noise.
|
||||
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
|
||||
if (this.speaking) {
|
||||
this.quietSamples += count;
|
||||
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
function getContext(): android.content.Context | null {
|
||||
if (!global.isAndroid) return null;
|
||||
return Utils.android.getApplicationContext() ?? null;
|
||||
}
|
||||
|
||||
/** Schedule the durable Android alarm backing one on-glasses timer. */
|
||||
export function scheduleTimerNotification(timerId: number, triggerAtMs: number, durationLabel: string): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawTimerNotifications.schedule(context, timerId, triggerAtMs, durationLabel);
|
||||
}
|
||||
|
||||
/** Cancel both a pending alarm and any already-posted notification. */
|
||||
export function cancelTimerNotification(timerId: number): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawTimerNotifications.cancel(context, timerId);
|
||||
}
|
||||
|
||||
/** Prompt worker-side expiry path; Java deduplicates it against the alarm. */
|
||||
export function fireTimerNotification(timerId: number, durationLabel: string): void {
|
||||
const context = getContext();
|
||||
if (!context) return;
|
||||
com.faceclaw.app.FaceclawTimerNotifications.fireNow(context, timerId, durationLabel);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
|
||||
|
||||
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
|
||||
export class TimedTranscript {
|
||||
text = "";
|
||||
private endMs: number | undefined;
|
||||
private speaker: string | undefined;
|
||||
|
||||
copy(): TimedTranscript {
|
||||
const copy = new TimedTranscript();
|
||||
copy.text = this.text;
|
||||
copy.endMs = this.endMs;
|
||||
copy.speaker = this.speaker;
|
||||
return copy;
|
||||
}
|
||||
|
||||
append(token: TranscriptToken): void {
|
||||
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
|
||||
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
|
||||
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
|
||||
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
|
||||
} else {
|
||||
this.text += token.text;
|
||||
}
|
||||
if (token.text.trim()) {
|
||||
if (token.end_ms != null) this.endMs = token.end_ms;
|
||||
if (token.speaker != null) this.speaker = token.speaker;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
@@ -14,7 +16,7 @@ export type VoiceControlState = {
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
* Complete best transcript of the current utterance. REPLACE semantics —
|
||||
* render as-is, replacing any previous partial. Not a delta.
|
||||
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private readonly speechPauseListeners = new Set<() => void>();
|
||||
private readonly speechPause = new SpeechPauseDetector();
|
||||
private readonly speechEndListeners = new Set<() => void>();
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.speechEndListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Pauses in on-device recognition, for Transcribe's presentation only. */
|
||||
onSpeechPause(listener: () => void): () => void {
|
||||
this.speechPauseListeners.add(listener);
|
||||
return () => this.speechPauseListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("ptt", options);
|
||||
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller?.clearSpeakerVerification();
|
||||
}
|
||||
|
||||
this.speechPause.reset();
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
if (cloudClient) {
|
||||
this.cloudClient = cloudClient;
|
||||
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
|
||||
if (options.provider === "onboard") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: { text: string; isFinal: boolean }) =>
|
||||
this.emitTranscript(event.text, event.isFinal),
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
this.emitTranscript(event.text, event.isFinal, event),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
|
||||
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new SonioxSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new SonioxSttClient(config), apiKey);
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
|
||||
// "My voice only": a rejected session's final transcript is emptied so
|
||||
// nothing downstream (assistant, dictation) acts on another speaker's
|
||||
// words. The verification verdict is posted before the final, and cloud
|
||||
// finals arrive over the network later still, so the flag is settled.
|
||||
if (isFinal && this.verificationActive && this.verificationRejected) {
|
||||
text = "";
|
||||
metadata = undefined;
|
||||
}
|
||||
const event = { text, isFinal };
|
||||
const event = { ...metadata, text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user