Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-11-01 11:00:38 -08:00
commit fcea0b369a
218 files changed
+14438 -2259

No files matched your search

+93
View File
@@ -0,0 +1,93 @@
/**
* Platform-neutral ambient-light types and the CFW report decoder (settings
* field 105), shared by ambient-light.ts and ambient-light.ios.ts.
*/
export const ALS_REPORT_VERSION = 1;
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
export type AmbientLightReport = {
reason: AmbientLightReason;
/** The sensor driver is open (readings are live). */
opened: boolean;
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
passive: boolean;
/** The user's stock auto-brightness setting is on. */
autoBrightness: boolean;
/** The last passive read succeeded (only meaningful while passive). */
readOk: boolean;
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
status: number;
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
alsValue: number;
/** Max of the last five readings (what the stock curve is applied to). */
peak: number;
/** Brightness level (0..100) the stock curve would choose for `peak`. */
stockTarget: number;
/** Current brightness setting (0..100). */
brightness: number;
/** Row of the stock brightness curve (0..5) for `peak`. */
gear: number;
/** Stock learned scale factor, Q10 (1024 = 1.0). */
scaleQ10: number;
/** Firmware millisecond tick when the report was built. */
tickMs: number;
};
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
function rd16(b: Uint8Array, i: number): number {
return b[i] | (b[i + 1] << 8);
}
function rd32(b: Uint8Array, i: number): number {
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
}
/** Decode a field-105 record body, or null when it isn't one. */
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
return null;
}
const flags = body[4];
return {
reason: REASONS[body[3]] ?? "poll",
opened: (flags & 0x01) !== 0,
passive: (flags & 0x02) !== 0,
autoBrightness: (flags & 0x04) !== 0,
readOk: (flags & 0x08) !== 0,
status: body[5],
alsValue: rd32(body, 6),
peak: rd32(body, 10),
stockTarget: rd16(body, 14),
brightness: body[16],
gear: body[17],
scaleQ10: rd16(body, 18),
tickMs: rd32(body, 20),
};
}
export type AmbientLightPollingOptions = {
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
intervalMs?: number;
/** Report when the reading moved by at least this much. Default 1. */
minDelta?: number;
/** Also report after this many ms without one (0 = never). Default 5000. */
heartbeatMs?: number;
/**
* Stop automatically when the Faceclaw framebuffer lease is released or
* lapses (the phone went away). Default true.
*/
bindToLease?: boolean;
};
/** The options with their defaults filled in. */
export function resolvePollingOptions(options: AmbientLightPollingOptions): Required<AmbientLightPollingOptions> {
return {
intervalMs: options.intervalMs ?? 500,
minDelta: options.minDelta ?? 1,
heartbeatMs: options.heartbeatMs ?? 5000,
bindToLease: options.bindToLease ?? true,
};
}
+51
View File
@@ -0,0 +1,51 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
import {
decodeAmbientLightReport,
resolvePollingOptions,
type AmbientLightPollingOptions,
type AmbientLightReport,
} from "./ambient-light-types";
export * from "./ambient-light-types";
/**
* Ambient light sensor access on iOS: the same API as ambient-light.ts (see
* there for the CFW query and passive-polling semantics) over the running
* shared-Kotlin session. No-ops while no session is running.
*/
/** Ask the firmware for one ambient-light report (delivered to listeners). */
export function queryAmbientLight(): void {
try {
FaceclawCommunicatorBridge.getActive()?.queryAmbientLight();
} catch (error) {
console.warn(`queryAmbientLight failed: ${error}`);
}
}
/** Start or stop CFW passive light-sensor polling. */
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
try {
FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
} catch (error) {
console.warn(`setAmbientLightPolling failed: ${error}`);
}
}
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
const active = FaceclawCommunicatorBridge.getActive();
if (!active) return () => {};
const unsubscribe = active.addAmbientLightListener((body) => {
const report = decodeAmbientLightReport(body);
if (report) listener(report);
});
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a disconnect.
}
};
}
+9 -85
View File
@@ -16,88 +16,17 @@
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
*/
import { toUint8Array } from "../util/array-util";
import {
decodeAmbientLightReport,
resolvePollingOptions,
type AmbientLightPollingOptions,
type AmbientLightReport,
} from "./ambient-light-types";
declare const com: any;
declare const global: any;
export const ALS_REPORT_VERSION = 1;
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
export type AmbientLightReport = {
reason: AmbientLightReason;
/** The sensor driver is open (readings are live). */
opened: boolean;
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
passive: boolean;
/** The user's stock auto-brightness setting is on. */
autoBrightness: boolean;
/** The last passive read succeeded (only meaningful while passive). */
readOk: boolean;
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
status: number;
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
alsValue: number;
/** Max of the last five readings (what the stock curve is applied to). */
peak: number;
/** Brightness level (0..100) the stock curve would choose for `peak`. */
stockTarget: number;
/** Current brightness setting (0..100). */
brightness: number;
/** Row of the stock brightness curve (0..5) for `peak`. */
gear: number;
/** Stock learned scale factor, Q10 (1024 = 1.0). */
scaleQ10: number;
/** Firmware millisecond tick when the report was built. */
tickMs: number;
};
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
function rd16(b: Uint8Array, i: number): number {
return b[i] | (b[i + 1] << 8);
}
function rd32(b: Uint8Array, i: number): number {
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
}
/** Decode a field-105 record body, or null when it isn't one. */
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
return null;
}
const flags = body[4];
return {
reason: REASONS[body[3]] ?? "poll",
opened: (flags & 0x01) !== 0,
passive: (flags & 0x02) !== 0,
autoBrightness: (flags & 0x04) !== 0,
readOk: (flags & 0x08) !== 0,
status: body[5],
alsValue: rd32(body, 6),
peak: rd32(body, 10),
stockTarget: rd16(body, 14),
brightness: body[16],
gear: body[17],
scaleQ10: rd16(body, 18),
tickMs: rd32(body, 20),
};
}
export type AmbientLightPollingOptions = {
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
intervalMs?: number;
/** Report when the reading moved by at least this much. Default 1. */
minDelta?: number;
/** Also report after this many ms without one (0 = never). Default 5000. */
heartbeatMs?: number;
/**
* Stop automatically when the Faceclaw framebuffer lease is released or
* lapses (the phone went away). Default true.
*/
bindToLease?: boolean;
};
export * from "./ambient-light-types";
function activeCommunicator(): any {
if (!global.isAndroid) return null;
@@ -123,14 +52,9 @@ export function queryAmbientLight(): void {
* on every session start.
*/
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
try {
activeCommunicator()?.setAmbientLightPolling(
enabled,
options.intervalMs ?? 500,
options.minDelta ?? 1,
options.heartbeatMs ?? 5000,
options.bindToLease ?? true,
);
activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
} catch (error) {
console.warn(`setAmbientLightPolling failed: ${error}`);
}
+50
View File
@@ -0,0 +1,50 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/**
* Passive view of the glasses' audio traffic on iOS: the same API as
* audio-monitor.ts over the running shared-Kotlin session. Nothing here sends
* anything to the glasses.
*/
export type AudioArm = "L" | "R" | "?";
/** The session listeners attach to; changes across reconnects. */
export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null {
return FaceclawCommunicatorBridge.getActive();
}
/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */
export function isFaceclawAudioCaptureActive(): boolean {
try {
return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive());
} catch {
return false;
}
}
/**
* Subscribe `communicator` to every audio-characteristic packet and every
* field-104 mic status record body (undecoded). Delivered on the main thread.
* Returns an unsubscribe function.
*/
export function addAudioMonitorListeners(
communicator: FaceclawCommunicatorBridge | null,
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
onStatus: (body: Uint8Array, arm: AudioArm) => void,
): () => void {
if (!communicator) return () => {};
const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs));
const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm)));
return () => {
try {
offPackets();
offStatus();
} catch {
// The session may have been closed during a disconnect.
}
};
}
function toArm(arm: string): AudioArm {
return arm === "L" || arm === "R" ? arm : "?";
}
+49
View File
@@ -0,0 +1,49 @@
import type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
export type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
// The iOS twin of the Android benchmark bridge. iOS has no static active
// communicator, so the preview controller binds its session here on connect
// and unbinds it on close; the run itself is the same shared-Kotlin
// GlassesSessionCore stream as on Android.
type BenchmarkSession = {
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean
cancelBandwidthBenchmark(): void
getBandwidthBenchmarkStatus(): string
}
let session: BenchmarkSession | null = null
export function bindBandwidthBenchmarkSession(next: BenchmarkSession | null): void {
session = next
}
/** Start a run. False if not connected or a run is already active. */
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
if (!session) return false
try {
return session.startBandwidthBenchmark(
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode))
} catch (error) {
console.warn(`startBandwidthBenchmark failed: ${error}`)
return false
}
}
/** Cancel an in-progress run. Safe to call when idle. */
export function cancelBandwidthBenchmark(): void {
try {
session?.cancelBandwidthBenchmark()
} catch (error) {
console.warn(`cancelBandwidthBenchmark failed: ${error}`)
}
}
/** Status of the current or most recent run; null when not connected. */
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
if (!session) return null
try {
return JSON.parse(session.getBandwidthBenchmarkStatus()) as BandwidthBenchmarkStatus
} catch (error) {
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`)
return null
}
}
+3
View File
@@ -22,6 +22,9 @@ export type BandwidthBenchmarkStatus = {
payloadBytesAcked: number;
wireBytesAcked: number;
aborted: boolean;
/** Each arm's current PHY ("2M", "1M", ...); null when the platform doesn't report it. */
leftPhy?: string | null;
rightPhy?: string | null;
};
function activeCommunicator(): any {
+9 -2
View File
@@ -2,11 +2,18 @@ import { Http } from '@nativescript/core';
import { withUserAgent } from '../util/http';
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
/**
* Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies.
*
* toArrayBuffer() is interop.bufferFromData, which in @nativescript/ios 9.0.3
* views the NSData's bytes without retaining it: once the response is
* collected, the buffer is dangling (a store EHPK was saved as 41 MB of
* zeros). Copy while the response is still alive.
*/
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
const response = await Http.request({ url, method: 'GET', headers: withUserAgent(), timeout: timeoutMs });
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()).slice(), contentLength: Number(length ?? 0) };
}
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
+11 -10
View File
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
/**
* Spoken language the provider should expect (ISO 639-1); empty lets it
* detect the language. Cloud providers only.
*/
language?: string;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
language: selection.language ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
}
/**
* What a capture transcribes with: the provider, and for Whisper server the
* language it is told to expect. A running capture is shared only by holders
* asking for the same.
* What a capture transcribes with: the provider and the language it is told
* to expect. A running capture is shared only by holders asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
return selection.provider === "whisper-server"
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
: selection.provider;
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
@@ -105,7 +106,7 @@ export function createCloudSttClient(
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
{ ...options, apiKey, language: (selection.language ?? "").trim() },
continuous,
provider.finalTimeoutMs,
);
+2
View File
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
export type CloudSttOptions = {
apiKey: string;
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
language?: string;
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
+4 -1
View File
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
start(): void {
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
// Without language_code the model detects the language, which can go
// badly wrong on short or noisy segments (Japanese heard as Russian).
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
const listener = {
onOpen: () => {
if (this.closed) return;
+124 -1
View File
@@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input";
import { File, knownFolders, path, type ImageSource } from "@nativescript/core";
import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat";
import { type CompassEvent } from "./compass-types";
import { type ImuReading } from "./imu-types";
import { fromData, toData } from "./kotlin-data";
import { previewPixels } from "./ios-graphics";
import { iosBluetooth } from "./ios-bluetooth";
@@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any;
declare const FaceclawKitIosAncsListener: any;
declare const FaceclawKitIosAudioPacketListener: any;
declare const FaceclawKitFaceclawCompassListener: any;
declare const FaceclawKitFaceclawImuListener: any;
declare const FaceclawKitIosAmbientLightListener: any;
declare const FaceclawKitIosMicStatusListener: any;
export type CommunicatorPhase =
@@ -164,6 +168,29 @@ class CompassListener extends NSObject {
}
}
@NativeClass()
class ImuListener extends NSObject {
static ObjCProtocols = [FaceclawKitFaceclawImuListener];
handler!: (reading: ImuReading) => void;
onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void {
this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) });
}
}
@NativeClass()
class AmbientLightListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosAmbientLightListener];
handler!: (body: Uint8Array) => void;
onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); }
}
@NativeClass()
class MicStatusListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosMicStatusListener];
handler!: (body: Uint8Array, arm: string) => void;
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
}
/**
* The iOS twin of the Android communicator bridge: the same public API over
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
@@ -173,6 +200,8 @@ class CompassListener extends NSObject {
* Android so callers never re-enter the bridge from a Kotlin frame.
*/
export class FaceclawCommunicatorBridge {
// The started, not yet closed bridge (see getActive).
private static active: FaceclawCommunicatorBridge | null = null;
private readonly communicator: any;
private readonly listenerProxy: SessionListener;
private ancsProxy: AncsListener | null = null;
@@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge {
private audioListener: AudioPacketListener | null = null;
private microphoneWork: Promise<void> = Promise.resolve();
private microphoneToken = 0;
private closed = false;
/**
* The bridge whose session is running, for modules that reach the glasses
* without the bridge threaded through (imu, ambient-light, audio-monitor):
* the iOS stand-in for Android's FaceclawBleCommunicator.getActive().
* Set by start(), cleared by close().
*/
static getActive(): FaceclawCommunicatorBridge | null {
return FaceclawCommunicatorBridge.active;
}
constructor(addresses: { right: string; left: string; ring?: string; ringRole?: string }) {
this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress(
@@ -454,7 +494,9 @@ export class FaceclawCommunicatorBridge {
// ----- session ---------------------------------------------------------------------
async start(): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.start(); });
await this.enqueueNativeCall(() => {
if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this;
});
}
async setG2ScreenOn(screenOn: boolean): Promise<void> {
@@ -644,6 +686,8 @@ export class FaceclawCommunicatorBridge {
}
async close(): Promise<void> {
this.closed = true;
if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null;
this.setAncsListeners(false);
this.previewAnimationListeners.clear();
await this.enqueueNativeCall(() => {
@@ -667,6 +711,85 @@ export class FaceclawCommunicatorBridge {
return () => { this.communicator.removeCompassListenerListener(proxy); };
}
/** Developer > Debug tests > BLE bandwidth; see app/native/bandwidth-benchmark.ios.ts. */
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean {
return Boolean(this.communicator.startBandwidthBenchmarkWithLinkModeMessageSizeWindowSizeDurationMsLinkMode(
messageSize, windowSize, durationMs, linkMode));
}
cancelBandwidthBenchmark(): void {
this.communicator.cancelBandwidthBenchmark();
}
/** Status/results of the current or most recent run, as JSON. */
getBandwidthBenchmarkStatus(): string {
return String(this.communicator.getBandwidthBenchmarkStatus());
}
// ----- sensors and passive audio, for the developer demos -----------------------------
// Fire-and-forget, so they skip the native-call queue like the compass controls do.
/** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */
setImuReportEnabled(enable: boolean, reportFrq: number): void {
this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq)));
}
/** Accelerometer samples, delivered on the main queue. */
addImuListener(listener: (reading: ImuReading) => void): () => void {
const proxy = ImuListener.new() as ImuListener;
proxy.handler = listener;
this.communicator.addImuListenerListener(proxy);
return () => { this.communicator.removeImuListenerListener(proxy); };
}
/** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */
queryAmbientLight(): void {
this.communicator.queryAmbientLight();
}
setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void {
this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable),
Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease));
}
/** Raw field-105 ambient-light records, delivered on the main queue. */
addAmbientLightListener(listener: (body: Uint8Array) => void): () => void {
const proxy = AmbientLightListener.new() as AmbientLightListener;
proxy.handler = listener;
const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy);
return () => { this.communicator.removeAmbientLightListenerListener(handle); };
}
/** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */
addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void {
const proxy = MicStatusListener.new() as MicStatusListener;
proxy.handler = listener;
const handle = this.communicator.addMicStatusListenerNsDataListener(proxy);
return () => { this.communicator.removeMicStatusListenerListener(handle); };
}
/** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */
sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean {
return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple(
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
}
/**
* Every audio-characteristic packet, whatever enabled the mics, on the
* main queue. Passive: sends nothing and doesn't affect capture.
*/
addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void {
const proxy = AudioPacketListener.new() as AudioPacketListener;
proxy.handler = listener;
const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy);
return () => { this.communicator.removeAudioMonitorListenerListener(handle); };
}
/** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */
isAudioCaptureActive(): boolean {
return Boolean(this.communicator.isAudioCaptureActive());
}
/**
* Serialized glasses microphone enable/disable (the voice bridge's
* IosMicrophoneSession contract). A release while an enable awaits its ACK
+17
View File
@@ -0,0 +1,17 @@
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
export type ImuReading = { x: number; y: number; z: number; source: number };
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
case 1:
return "R arm";
case 2:
return "ring";
case 3:
return "L arm";
default:
return "device";
}
}
+34
View File
@@ -0,0 +1,34 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
import { type ImuReading } from "./imu-types";
export * from "./imu-types";
/**
* IMU (accelerometer) access on iOS: the same API as imu.ts over the running
* shared-Kotlin session. As there, the stream is experimental, the calls are
* no-ops while no session is running, and a listener stays with the session
* it subscribed to.
*/
/** Enable/disable the accelerometer report stream. No-op if not connected. */
export function setImuReportEnabled(enable: boolean, reportFreq: number): void {
try {
FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq);
} catch (error) {
console.warn(`setImuReportEnabled failed: ${error}`);
}
}
/** Subscribe to IMU readings; returns an unsubscribe function. */
export function addImuListener(listener: (reading: ImuReading) => void): () => void {
const active = FaceclawCommunicatorBridge.getActive();
if (!active) return () => {};
const unsubscribe = active.addImuListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may be gone after a disconnect; nothing to clean up.
}
};
}
+3 -15
View File
@@ -12,7 +12,9 @@ declare const com: any;
* static, so it works from any isolate without threading the bridge through.
*/
export type ImuReading = { x: number; y: number; z: number; source: number };
import { type ImuReading } from "./imu-types";
export * from "./imu-types";
function activeCommunicator(): any {
if (!global.isAndroid) return null;
@@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v
}
};
}
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
case 1:
return "R arm";
case 2:
return "ring";
case 3:
return "L arm";
default:
return "device";
}
}
+4 -6
View File
@@ -6,12 +6,14 @@ import type {
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { buildQwenPrompt } from "../assistant/qwen-prompt";
import type { LocalModelState } from "./local-llm-types";
declare const com: any;
declare const java: any;
/**
* On-phone LLM backend: model download management plus a streaming client
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
* local-llm.ios.ts instead): model download management plus a streaming client
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
* the direct assistant loop can treat "local" as just another provider.
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
@@ -38,11 +40,7 @@ const TOP_K = 20;
/** Free the ~3GB of model + KV this long after the last generation. */
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
export type LocalModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
export type { LocalModelState } from "./local-llm-types";
let runner: any = null;
let downloader: any = null;
+13
View File
@@ -0,0 +1,13 @@
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
export type LocalModelState = {
/**
* Android downloads its model: absent → downloading → ready. iOS uses the
* system's Apple Intelligence model: ready, or unavailable (see detail).
*/
status: "absent" | "downloading" | "ready" | "unavailable";
bytesDownloaded: number;
totalBytes: number;
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
detail?: string;
};
+383
View File
@@ -0,0 +1,383 @@
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import type { LocalModelState } from "./local-llm-types";
export type { LocalModelState } from "./local-llm-types";
declare const FaceclawSystemLanguageModel: any;
/**
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
* Calculator and the Microphones helpers treat it as the "local" provider.
*
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
* background (where Faceclaw usually is, driving the glasses with the phone
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
* ~3 GB in-process model makes the app the first thing jetsam kills. The
* system model runs out of process on the Neural Engine, needs no download,
* and decodes tool arguments under a schema constraint (like our GBNF grammar
* on Android). Its costs: a 4096-token context (the Swift side fits history
* and tool definitions into it), Apple's guardrails, a background request
* budget, and Apple Intelligence-capable hardware.
*
* Tool calls: the framework runs tools itself, mid-response. The Swift side
* parks each call and reports it; here that ends the loop iteration with
* stopReason "tool_use", the direct backend executes the tools as for any
* provider, and its next request (ending in the matching tool_result blocks)
* resumes the parked generation instead of starting a new one.
*/
export const LOCAL_MODEL = {
id: "apple-foundation-model",
label: "Apple Intelligence",
sizeBytes: 0,
};
const DEFAULT_MAX_TOKENS = 512;
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
/**
* Only used when tool results arrive with no parked generation to resume (it
* timed out, or the app restarted mid-turn): the framework has no way to
* continue after a tool output without a new prompt.
*/
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
type Availability =
| "available"
| "unsupportedOS"
| "deviceNotEligible"
| "appleIntelligenceNotEnabled"
| "modelNotReady"
| "unavailable";
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "needs iOS 26",
deviceNotEligible: "not on this iPhone",
appleIntelligenceNotEnabled: "Apple Intelligence off",
modelNotReady: "downloading",
unavailable: "unavailable",
};
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
appleIntelligenceNotEnabled:
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
};
export function localModelAvailability(): Availability {
try {
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
const value = String(FaceclawSystemLanguageModel.availability());
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
} catch {
return "unsupportedOS";
}
}
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
export function isLocalModelSupported(): boolean {
const availability = localModelAvailability();
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
}
export function isLocalModelReady(): boolean {
return localModelAvailability() === "available";
}
export function localModelState(): LocalModelState {
const availability = localModelAvailability();
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
}
/** The user-facing reason the model can't run, or "" when it can. */
export function localModelUnavailableMessage(): string {
const availability = localModelAvailability();
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
}
// iOS manages the system model itself: there is nothing to download, cancel,
// delete or unload, and no progress to watch.
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
return () => {};
}
export function startLocalModelDownload(): void {}
export function cancelLocalModelDownload(): void {}
export function deleteLocalModel(): void {}
export function unloadLocalModel(): void {}
type HistoryEntry =
| { type: "prompt" | "response"; text: string }
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
| { type: "toolOutput"; id: string; name: string; text: string };
export type SystemModelRequest = {
system: string;
tools: Array<{ name: string; description: string; parameters: object }>;
history: HistoryEntry[];
prompt: string;
maxTokens: number;
permissive: boolean;
};
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
/**
* Translate provider-neutral history into the Swift request: everything but
* the newest user prompt becomes Transcript entries, and that prompt is what
* the model answers.
*/
export function buildSystemModelRequest(
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
): SystemModelRequest {
const messages = options.messages;
const last = messages[messages.length - 1];
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
const history: HistoryEntry[] = [];
const toolNames = new Map<string, string>();
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
appendHistory(history, message, toolNames);
}
const tools = (options.tools ?? []).map((tool) => ({
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
}));
return {
system: options.system?.trim() || "You are a helpful assistant.",
tools,
history,
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
permissive: tools.length === 0,
};
}
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
if (message.role === "user") {
for (const result of toolResultsOf(message)) {
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
}
const text = textOf(message.content);
if (text) history.push({ type: "prompt", text });
return;
}
const text = textOf(message.content);
if (text) history.push({ type: "response", text });
if (typeof message.content === "string") return;
const calls: Array<{ id: string; name: string; arguments: string }> = [];
for (const block of message.content) {
if (block.type !== "tool_use") continue;
toolNames.set(block.id, block.name);
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
}
if (calls.length) history.push({ type: "toolCalls", calls });
}
function textOf(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content.trim();
return content
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
.map((block) => block.text.trim())
.filter(Boolean)
.join("\n");
}
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
if (!message || message.role !== "user" || typeof message.content === "string") return [];
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
}
function parseArguments(json: string): any {
try {
const value = JSON.parse(json);
return value && typeof value === "object" ? value : {};
} catch {
return {};
}
}
type NativeEvent = {
kind: string;
text?: string;
calls?: Array<{ id: string; name: string; arguments: string }>;
code?: string;
message?: string;
};
/** Generations parked in tool calls, by call id. */
const parked = new Map<string, Generation>();
/** Every unfinished generation, so its native object outlives the caller's handle. */
const live = new Set<Generation>();
/**
* One native generation, possibly spanning several loop iterations: each
* iteration ends at a batch of tool calls (or the end of the reply), and the
* next one resumes it with the results.
*/
class Generation {
private readonly native: any;
private options: LlmStreamOptions | null;
private iteration = 0;
private text = "";
private callIds: string[] = [];
private expiry: ReturnType<typeof setTimeout> | null = null;
private closed = false;
constructor(options: LlmStreamOptions) {
this.options = options;
this.native = FaceclawSystemLanguageModel.new();
this.native.eventHandler = (json: string) => this.onEvent(String(json));
live.add(this);
}
start(request: SystemModelRequest): LlmStreamHandle {
this.native.start(JSON.stringify(request));
return this.handle();
}
/** Whether these results answer this generation's parked calls. */
awaits(results: ToolResultBlock[]): boolean {
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
}
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
this.unpark();
this.options = options;
this.iteration++;
this.text = "";
this.native.resolveToolCalls(
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
);
return this.handle();
}
cancel(): void {
if (this.closed) return;
this.close();
try {
this.native.cancel();
} catch {
// ignore
}
}
private handle(): LlmStreamHandle {
const iteration = this.iteration;
// A stale handle (an earlier iteration's) still cancels a generation parked
// since then: its owner gave up on the turn.
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
}
private close(): void {
this.closed = true;
this.unpark();
this.options = null;
this.native.eventHandler = null;
live.delete(this);
}
private park(ids: string[]): void {
this.callIds = ids;
for (const id of ids) parked.set(id, this);
this.expiry = setTimeout(() => {
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
this.cancel();
}, PARKED_TIMEOUT_MS);
}
private unpark(): void {
for (const id of this.callIds) {
if (parked.get(id) === this) parked.delete(id);
}
this.callIds = [];
if (this.expiry !== null) clearTimeout(this.expiry);
this.expiry = null;
}
private onEvent(json: string): void {
if (this.closed) return;
let event: NativeEvent;
try {
event = JSON.parse(json);
} catch {
return;
}
if (event.kind === "log") {
console.log(`[local-llm] ${event.message ?? ""}`);
return;
}
const options = this.options;
if (!options) return;
switch (event.kind) {
case "delta": {
const delta = event.text ?? "";
this.text += delta;
options.onTextDelta?.(delta, this.text);
return;
}
case "toolCalls": {
const calls = event.calls ?? [];
const text = this.text.trim();
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
for (const call of calls) {
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
}
this.options = null;
this.park(calls.map((call) => call.id));
options.onDone({ text, content, stopReason: "tool_use" });
return;
}
case "done": {
this.close();
const text = this.text.trim();
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
return;
}
case "error": {
this.close();
if (event.code === "refusal") {
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
} else {
options.onError(event.message || "The on-device model failed");
}
return;
}
}
}
}
/**
* Stream a reply from the system model. A request whose history ends in tool
* results for a parked generation resumes it; anything else starts afresh.
*/
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
const availability = localModelAvailability();
if (availability !== "available") {
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
return { cancel: () => {} };
}
const results = toolResultsOf(options.messages[options.messages.length - 1]);
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
if (waiting?.awaits(results)) return waiting.resume(options, results);
let generation: Generation | null = null;
try {
generation = new Generation(options);
return generation.start(buildSystemModelRequest(options));
} catch (error) {
generation?.cancel();
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
}
+28
View File
@@ -0,0 +1,28 @@
/**
* The on-phone LLM: one entry point for every feature that can run on the
* phone without an API key (the direct assistant's "local" provider,
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
* contract, plus model status for Settings.
*
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
* to download, available on Apple Intelligence devices running iOS 26+.
*/
export {
LOCAL_MODEL,
isLocalModelReady,
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
cancelLocalModelDownload,
deleteLocalModel,
unloadLocalModel,
streamLocalQwen as streamLocalModel,
} from "./llama";
export type { LocalModelState } from "./local-llm-types";
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
export function isLocalModelSupported(): boolean {
return true;
}
+43 -2
View File
@@ -6,6 +6,9 @@ import {
type MediaBrowserApp,
type MediaBrowserBridge,
} from "./media-types";
import { spotifyService } from "./spotify.ios";
import type { SpotifyLibrary } from "./spotify-library";
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
@@ -30,7 +33,7 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
connect(app: MediaBrowserApp): Promise<string> {
this.disconnect();
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS"));
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Not an Apple Music library"));
if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed"));
this.connected = true;
return Promise.resolve("root");
@@ -67,4 +70,42 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
}
}
export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge();
/** Apple Music's library, plus the Spotify library once signed in. */
export class IosMediaBrowserBridge implements MediaBrowserBridge {
private current: MediaBrowserBridge | null = null;
constructor(
private readonly appleMusic: AppleMusicBrowserBridge,
private readonly spotify: () => SpotifyLibrary | null,
) {}
listBrowsableApps(): MediaBrowserApp[] {
return [...this.appleMusic.listBrowsableApps(), ...(this.spotify()?.listBrowsableApps() ?? [])];
}
connect(app: MediaBrowserApp): Promise<string> {
this.disconnect();
const library = app.packageName === SPOTIFY_IOS_BUNDLE_ID ? this.spotify() : this.appleMusic;
if (!library) return Promise.reject(new Error(`${app.appName} can't be browsed`));
this.current = library;
return library.connect(app);
}
browse(parentId: string): Promise<MediaBrowseItem[]> {
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
}
playFromMediaId(mediaId: string): void {
this.current?.playFromMediaId(mediaId);
}
disconnect(): void {
this.current?.disconnect();
this.current = null;
}
}
export const mediaBrowserBridge: MediaBrowserBridge = new IosMediaBrowserBridge(
new AppleMusicBrowserBridge(),
() => spotifyService()?.library ?? null,
);
+54 -1
View File
@@ -2,6 +2,9 @@ import { Utils } from "@nativescript/core";
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
import { onSettingsStoreChanged } from "./settings-store";
import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types";
import { spotifyService } from "./spotify";
import type { SpotifyLibrary } from "./spotify-library";
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
declare const com: any;
@@ -157,4 +160,54 @@ export class FaceclawMediaBrowserBridge implements MediaBrowserBridge {
}
}
export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge();
/**
* The Android libraries: apps' media browser services, with Spotify's
* library from its Web API instead. Spotify's own browser service admits
* only allowlisted callers (Android Auto, Wear, Assistant), so connecting to
* it always fails; signed in, its row opens the Web API library, and signed
* out, opening it says how to sign in. Signed in with Spotify not installed,
* the library is still offered (it plays on the user's other devices).
*/
export class AndroidMediaBrowserBridge implements MediaBrowserBridge {
private current: MediaBrowserBridge | null = null;
constructor(
private readonly services: MediaBrowserBridge,
private readonly spotify: () => SpotifyLibrary | null,
private readonly isEnabled: (packageName: string) => boolean = isMediaAppEnabled,
) {}
listBrowsableApps(refresh = false): MediaBrowserApp[] {
const apps = this.services.listBrowsableApps(refresh);
const library = this.spotify();
if (!library || apps.some((app) => app.packageName === SPOTIFY_ANDROID_PACKAGE) || !this.isEnabled(SPOTIFY_ANDROID_PACKAGE)) {
return apps;
}
return [...apps, ...library.listBrowsableApps()];
}
connect(app: MediaBrowserApp): Promise<string> {
this.disconnect();
const library = app.packageName === SPOTIFY_ANDROID_PACKAGE ? this.spotify() : null;
this.current = library ?? this.services;
return this.current.connect(library ? library.app : app);
}
browse(parentId: string): Promise<MediaBrowseItem[]> {
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
}
playFromMediaId(mediaId: string): void {
this.current?.playFromMediaId(mediaId);
}
disconnect(): void {
this.current?.disconnect();
this.current = null;
}
}
export const mediaBrowserBridge: MediaBrowserBridge = new AndroidMediaBrowserBridge(
new FaceclawMediaBrowserBridge(),
() => spotifyService()?.library ?? null,
);
+195 -5
View File
@@ -11,6 +11,8 @@ import {
type MediaPlaybackState,
type MediaQueueItem,
} from "./media-types";
import { spotifyService } from "./spotify.ios";
import type { SpotifyPlayer } from "./spotify-player";
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
@@ -88,7 +90,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
onStateChange(listener: (state: MediaControllerState) => void): () => void {
void this.start();
this.stateListeners.add(listener);
this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS);
this.pollTimer ??= setInterval(() => this.publish(), POLL_MS);
listener(this.snapshot());
return () => {
this.stateListeners.delete(listener);
@@ -142,7 +144,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
if (this.hasAccess()) return Promise.resolve(true);
this.pendingAccess ??= this.askForAccess().finally(() => {
this.pendingAccess = null;
this.refresh();
this.publish();
});
return this.pendingAccess;
}
@@ -187,12 +189,12 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
const music = FaceclawAppleMusic.shared();
if (!this.bound) {
this.bound = true;
music.changeHandler = () => this.refresh();
music.changeHandler = () => this.publish();
}
return music;
}
private refresh(): void {
private publish(): void {
if (!this.stateListeners.size) return;
const state = this.snapshot();
const signature = stateSignature(state);
@@ -228,4 +230,192 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
}
}
export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge();
export type IosMediaSource = "apple-music" | "spotify";
/**
* Which player to follow: the one playing; with both playing, the current
* choice (no flapping); with neither, the one that played last, and before
* either has played this run, Spotify when it reports a session (its player
* reports only a recently active one, while the system Music player keeps
* its last item for days).
*/
export function chooseMediaSource(
current: IosMediaSource,
appleMusic: MediaControllerState,
spotify: MediaControllerState | null,
appleMusicLastPlayingAtMs: number,
spotifyLastPlayingAtMs: number,
): IosMediaSource {
if (!spotify?.accessEnabled) return "apple-music";
if (!appleMusic.accessEnabled) return "spotify";
const applePlaying = appleMusic.playbackState === "playing";
const spotifyPlaying = spotify.playbackState === "playing";
if (applePlaying && spotifyPlaying) return current;
if (applePlaying) return "apple-music";
if (spotifyPlaying) return "spotify";
if (appleMusicLastPlayingAtMs !== spotifyLastPlayingAtMs) {
return spotifyLastPlayingAtMs > appleMusicLastPlayingAtMs ? "spotify" : "apple-music";
}
return spotify.available ? "spotify" : "apple-music";
}
/**
* The iOS Music bridge: Apple Music (the system player), plus Spotify once
* signed in (its Web API: playback on this phone or any Spotify Connect
* device). The state and every command follow one of them, by
* chooseMediaSource.
*/
export class IosMediaControllerBridge implements MediaControllerBridge {
readonly accessPrompt =
"Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone. For Spotify, sign in from the app menu instead.";
readonly accessLabel: string;
readonly accessAction: string;
private source: IosMediaSource = "apple-music";
private appleMusicLastPlayingAtMs = 0;
private readonly listeners = new Set<(state: MediaControllerState) => void>();
private unsubscribeSources: Array<() => void> = [];
/** The source and its artVersion the composite artVersion was bumped for. */
private artKey = "";
private artVersion = 0;
constructor(
private readonly appleMusic: AppleMusicControllerBridge,
private readonly spotify: () => SpotifyPlayer | null,
private readonly now: () => number = Date.now,
) {
this.accessLabel = appleMusic.accessLabel;
this.accessAction = appleMusic.accessAction;
}
get skipRebuildsQueue(): boolean {
return this.active().skipRebuildsQueue;
}
onStateChange(listener: (state: MediaControllerState) => void): () => void {
if (!this.listeners.size) {
const update = () => this.emit();
const spotify = this.spotify();
this.unsubscribeSources = [this.appleMusic.onStateChange(update), ...(spotify ? [spotify.onStateChange(update)] : [])];
}
this.listeners.add(listener);
listener(this.snapshot());
return () => {
this.listeners.delete(listener);
if (this.listeners.size) return;
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
this.unsubscribeSources = [];
};
}
snapshot(): MediaControllerState {
const appleMusic = this.appleMusic.snapshot();
const spotify = this.spotify()?.snapshot() ?? null;
this.select(appleMusic, spotify);
const state = this.source === "spotify" && spotify ? spotify : appleMusic;
// Changes with the source as well as with that source's art.
const artKey = `${this.source}:${state.artVersion}`;
if (artKey !== this.artKey) {
this.artKey = artKey;
this.artVersion++;
}
return {
...state,
status: state.available ? state.status : idleStatus(appleMusic, spotify, state),
artVersion: this.artVersion,
// A Spotify command can fail while Apple Music is the one shown (playing from the Spotify library).
notice: spotify?.notice || state.notice || "",
};
}
async start(): Promise<void> {
await this.appleMusic.start();
}
async stop(): Promise<void> {
await this.appleMusic.stop();
}
playPause(): Promise<void> {
return this.active().playPause();
}
skipNext(): Promise<void> {
return this.active().skipNext();
}
skipPrevious(): Promise<void> {
return this.active().skipPrevious();
}
/** Apple Music's media-library permission (what requestAccess asks for); Spotify signs in from the Music menu. */
hasAccess(): boolean {
return this.appleMusic.hasAccess();
}
requestAccess(): Promise<boolean> {
return this.appleMusic.requestAccess();
}
skipToQueueItem(id: number): Promise<void> {
return this.active().skipToQueueItem(id);
}
getMediaVolumePercent(): number {
return this.active().getMediaVolumePercent();
}
setMediaVolumePercent(volumePercent: number): void {
this.active().setMediaVolumePercent(volumePercent);
}
getAlbumArt(maxSize: number): GrayImage | null {
return this.active().getAlbumArt(maxSize);
}
getQueue(): MediaQueueItem[] {
return this.active().getQueue();
}
async sendCustomAction(action: string): Promise<void> {
await this.active().sendCustomAction?.(action);
}
async refresh(): Promise<void> {
await this.spotify()?.refresh();
}
private active(): MediaControllerBridge {
const spotify = this.spotify();
this.select(this.appleMusic.snapshot(), spotify?.snapshot() ?? null);
return this.source === "spotify" && spotify ? spotify : this.appleMusic;
}
private select(appleMusic: MediaControllerState, spotify: MediaControllerState | null): void {
if (appleMusic.playbackState === "playing") this.appleMusicLastPlayingAtMs = this.now();
this.source = chooseMediaSource(this.source, appleMusic, spotify, this.appleMusicLastPlayingAtMs,
this.spotify()?.lastPlayingAtMs ?? 0);
}
/**
* Forward every source update: each source already drops repeats, and
* Spotify also signals changes that are not in the state (its queue).
*/
private emit(): void {
if (!this.listeners.size) return;
const state = this.snapshot();
for (const listener of this.listeners) listener(state);
}
}
/** The no-session line: which players were checked. */
function idleStatus(appleMusic: MediaControllerState, spotify: MediaControllerState | null, shown: MediaControllerState): string {
if (appleMusic.accessEnabled && spotify?.accessEnabled) return "Nothing is playing.";
return shown.status;
}
export const appleMusicControllerBridge = new AppleMusicControllerBridge();
export const mediaControllerBridge: MediaControllerBridge = new IosMediaControllerBridge(
appleMusicControllerBridge,
() => spotifyService()?.player ?? null,
);
+251 -1
View File
@@ -12,11 +12,14 @@ import {
type MediaPlaybackState,
type MediaQueueItem,
} from "./media-types";
import { spotifyService } from "./spotify";
import { SPOTIFY_ANDROID_PACKAGE, type SpotifyPlayer } from "./spotify-player";
declare const com: any;
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
/** Android media sessions (any player that publishes one), through FaceclawMediaController. */
export class FaceclawMediaControllerBridge implements MediaControllerBridge {
readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings.";
readonly accessLabel = "Notification access needed";
@@ -215,4 +218,251 @@ function normalizePlaybackState(value: string): MediaPlaybackState {
}
}
export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge();
export type AndroidMediaSource = "session" | "spotify";
/**
* Which to follow: the media session (any player, Spotify included when it
* runs on this phone) or Spotify's Web API player (signed in; it also sees
* Spotify playing on a computer or speaker). The one playing; with both
* playing, the session when it is Spotify's own (the same playback), else the
* current choice (no flapping); with neither, a Spotify session, then the one
* that played last, then whichever has something. Without notification
* access the session shows its access prompt unless Spotify has something.
*/
export function chooseAndroidMediaSource(
current: AndroidMediaSource,
session: MediaControllerState,
spotify: MediaControllerState | null,
sessionLastPlayingAtMs: number,
spotifyLastPlayingAtMs: number,
): AndroidMediaSource {
if (!spotify?.accessEnabled) return "session";
if (!session.accessEnabled) return spotify.available ? "spotify" : "session";
const sessionIsSpotify = session.packageName === SPOTIFY_ANDROID_PACKAGE;
const sessionPlaying = session.playbackState === "playing";
const spotifyPlaying = spotify.playbackState === "playing";
if (sessionPlaying && spotifyPlaying) return sessionIsSpotify ? "session" : current;
if (sessionPlaying) return "session";
if (spotifyPlaying) return "spotify";
if (sessionIsSpotify && session.available) return "session";
if (sessionLastPlayingAtMs !== spotifyLastPlayingAtMs) {
return spotifyLastPlayingAtMs > sessionLastPlayingAtMs ? "spotify" : "session";
}
return session.available || !spotify.available ? "session" : "spotify";
}
/**
* The Android Music bridge: media sessions, plus Spotify's Web API once
* signed in (optional). Spotify playing on this phone stays the session's:
* its state, art, queue and transport come from the session (instant, no
* quota), and the Web API adds what the session lacks: shuffle / repeat /
* like rows and the volume of a Spotify Connect speaker or computer. Spotify
* playing on another device, with no session here, follows the Web API player.
*/
export class AndroidMediaControllerBridge implements MediaControllerBridge {
readonly accessPrompt: string;
readonly accessLabel: string;
readonly accessAction: string;
private source: AndroidMediaSource = "session";
private sessionLastPlayingAtMs = 0;
private readonly listeners = new Set<(state: MediaControllerState) => void>();
private unsubscribeSources: Array<() => void> = [];
/** The source and its artVersion the composite artVersion was bumped for. */
private artKey = "";
private artVersion = 0;
/** The player the last getQueue listed, so a jump goes to the same one. */
private queueSource: MediaControllerBridge | null = null;
/** A Spotify session's item and play state, to re-read the Web API when they change. */
private spotifySessionKey = "";
constructor(
private readonly session: MediaControllerBridge,
private readonly spotify: () => SpotifyPlayer | null,
private readonly now: () => number = Date.now,
) {
this.accessPrompt = session.accessPrompt;
this.accessLabel = session.accessLabel;
this.accessAction = session.accessAction;
}
get skipRebuildsQueue(): boolean {
return (this.queueSource ?? this.active()).skipRebuildsQueue;
}
onStateChange(listener: (state: MediaControllerState) => void): () => void {
if (!this.listeners.size) {
const spotify = this.spotify();
this.unsubscribeSources = [
this.session.onStateChange((state) => this.sessionChanged(state)),
...(spotify ? [spotify.onStateChange(() => this.emit())] : []),
];
}
this.listeners.add(listener);
listener(this.snapshot());
return () => {
this.listeners.delete(listener);
if (this.listeners.size) return;
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
this.unsubscribeSources = [];
};
}
snapshot(): MediaControllerState {
const session = this.session.snapshot();
const spotify = this.spotify()?.snapshot() ?? null;
this.select(session, spotify);
const state = this.source === "spotify" && spotify ? spotify : withSpotifyExtras(session, spotify);
// Changes with the source as well as with that source's art.
const artKey = `${this.source}:${state.artVersion}`;
if (artKey !== this.artKey) {
this.artKey = artKey;
this.artVersion++;
}
return {
...state,
artVersion: this.artVersion,
// A Spotify command can fail while the session is the one shown (playing from the Spotify library).
notice: spotify?.notice || state.notice || "",
};
}
async start(): Promise<void> {
await this.session.start();
}
async stop(): Promise<void> {
await this.session.stop();
}
playPause(): Promise<void> {
return this.active().playPause();
}
skipNext(): Promise<void> {
return this.active().skipNext();
}
skipPrevious(): Promise<void> {
return this.active().skipPrevious();
}
/** Notification access (what requestAccess opens); Spotify signs in from the Music menu. */
hasAccess(): boolean {
return this.session.hasAccess();
}
requestAccess(): Promise<boolean> {
return this.session.requestAccess();
}
skipToQueueItem(id: number): Promise<void> {
return (this.queueSource ?? this.active()).skipToQueueItem(id);
}
/** Spotify's own volume logic (a Connect device's volume, else this phone's) whenever Spotify is shown. */
getMediaVolumePercent(): number {
return this.volumeSource().getMediaVolumePercent();
}
setMediaVolumePercent(volumePercent: number): void {
this.volumeSource().setMediaVolumePercent(volumePercent);
}
getAlbumArt(maxSize: number): GrayImage | null {
return this.active().getAlbumArt(maxSize);
}
/** The session's queue; for a Spotify session that lists none, the Web API's. */
getQueue(): MediaQueueItem[] {
const active = this.active();
const queue = active.getQueue();
const spotify = this.spotifyExtras();
if (queue.length || active !== this.session || !spotify) {
this.queueSource = active;
return queue;
}
this.queueSource = spotify;
return spotify.getQueue();
}
/** Spotify's actions (shown with a Spotify session too) go to the Web API player; any others to the session. */
async sendCustomAction(action: string): Promise<void> {
const spotify = this.spotify();
if (spotify && spotify.snapshot().customActions?.some((custom) => custom.action === action)) {
await spotify.sendCustomAction(action);
} else {
await this.session.sendCustomAction?.(action);
}
}
async refresh(): Promise<void> {
await this.spotify()?.refresh();
}
private active(): MediaControllerBridge {
const spotify = this.spotify();
this.select(this.session.snapshot(), spotify?.snapshot() ?? null);
return this.source === "spotify" && spotify ? spotify : this.session;
}
/** The signed-in Web API player when it describes what the session shows (Spotify's own session); else null. */
private spotifyExtras(): SpotifyPlayer | null {
const spotify = this.spotify();
if (!spotify || this.session.snapshot().packageName !== SPOTIFY_ANDROID_PACKAGE) return null;
return spotify.snapshot().available ? spotify : null;
}
private volumeSource(): MediaControllerBridge {
const active = this.active();
return active === this.session ? this.spotifyExtras() ?? this.session : active;
}
private select(session: MediaControllerState, spotify: MediaControllerState | null): void {
if (session.playbackState === "playing") this.sessionLastPlayingAtMs = this.now();
this.source = chooseAndroidMediaSource(this.source, session, spotify, this.sessionLastPlayingAtMs,
this.spotify()?.lastPlayingAtMs ?? 0);
}
private sessionChanged(state: MediaControllerState): void {
// Spotify's session changed track or started / stopped: re-read the Web
// API now, so its rows (like this song, shuffle) follow at once rather
// than at the next poll. Signed out, the player skips the request.
const key = state.packageName === SPOTIFY_ANDROID_PACKAGE ? `${state.title}\n${state.playbackState}` : "";
if (key !== this.spotifySessionKey) {
this.spotifySessionKey = key;
if (key) void this.spotify()?.refresh();
}
this.emit();
}
/** Forward every source update: each drops its own repeats, and Spotify also signals queue changes. */
private emit(): void {
if (!this.listeners.size) return;
const state = this.snapshot();
for (const listener of this.listeners) listener(state);
}
}
/**
* Spotify's session state plus what only the Web API knows: its custom rows
* and, when it plays elsewhere, the device ("Spotify · Kitchen"). "Like
* song" names the Web API's current item, so it is left out until both agree
* on the track (the API lags a skip by up to a poll).
*/
export function withSpotifyExtras(session: MediaControllerState, spotify: MediaControllerState | null): MediaControllerState {
if (!spotify?.available || session.packageName !== SPOTIFY_ANDROID_PACKAGE) return session;
const sameItem = spotify.title === session.title;
const actions = (spotify.customActions ?? []).filter((custom) => sameItem || custom.action !== "like");
return {
...session,
appName: sameItem ? spotify.appName : session.appName,
customActions: [...(session.customActions ?? []), ...actions],
};
}
const sessionBridge = new FaceclawMediaControllerBridge();
export const mediaControllerBridge: MediaControllerBridge = new AndroidMediaControllerBridge(
sessionBridge,
() => spotifyService()?.player ?? null,
);
+23 -1
View File
@@ -3,7 +3,8 @@ import type { GrayImage } from "../graphics/image";
/**
* Shapes shared by the media bridges: media-controller.ts / media-browser.ts
* drive Android media sessions and MediaBrowserServices; their .ios.ts
* counterparts drive Apple Music through the MediaPlayer framework.
* counterparts drive Apple Music through the MediaPlayer framework. Both
* add Spotify through its Web API once signed in (spotify-*.ts).
*/
/** The player behind the iOS bridges: the system Music app. */
@@ -13,9 +14,19 @@ export const APPLE_MUSIC_PACKAGE = "com.apple.Music";
export type MediaQueueItem = {
id: number;
title: string;
/** Usually the artist; absent when the player gives none. */
subtitle?: string;
active: boolean;
};
/** A player-specific action beyond the transport controls (Spotify's shuffle, repeat, like). */
export type MediaCustomAction = {
/** Stable id to send back with sendCustomAction. */
action: string;
/** The row label, naming the current state where there is one ("Shuffle: on"). */
name: string;
};
export type MediaPlaybackState =
| "notification-access-required"
| "idle"
@@ -47,6 +58,10 @@ export type MediaControllerState = {
* players re-publish the same strings with a different bitmap.
*/
artVersion: number;
/** Player-specific actions (see sendCustomAction); absent when the player offers none. */
customActions?: MediaCustomAction[];
/** A short-lived message about a command that failed (Spotify needs Premium, ...); absent or "" otherwise. */
notice?: string;
};
export const DEFAULT_MEDIA_STATE: MediaControllerState = {
@@ -113,6 +128,13 @@ export interface MediaControllerBridge {
getAlbumArt(maxSize: number): GrayImage | null;
/** The player's queue (playlist), empty when the player exposes none. */
getQueue(): MediaQueueItem[];
/** Run one of the state's customActions. */
sendCustomAction?(action: string): Promise<void>;
/**
* Re-read the player now, for one-off readers (the assistant) when no
* listener keeps the state current; absent when snapshots are always live.
*/
refresh?(): Promise<void>;
}
export type MediaBrowserApp = {
+30
View File
@@ -0,0 +1,30 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/** The iOS twin of mic-control-link.ts, over the running shared-Kotlin session. */
export function micControlSession(): unknown {
return FaceclawCommunicatorBridge.getActive();
}
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
export function sendMicControlRecord(
session: unknown,
record: Uint8Array,
label: string,
rightTemple: boolean,
leftTemple: boolean,
): boolean {
return (session as FaceclawCommunicatorBridge).sendFaceclawMicControl(record, label, rightTemple, leftTemple);
}
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
const unsubscribe = (session as FaceclawCommunicatorBridge).addMicStatusListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a reconnect.
}
};
}
+46
View File
@@ -0,0 +1,46 @@
import { toJavaBytes } from "./cloud-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
/**
* Platform plumbing for the CFW mic_control channel (apps/microphones/mic-control.ts):
* the running session to send records on and to hear field-104 status replies
* from. The session is opaque and is a different object after a reconnect.
*/
export function micControlSession(): unknown {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
export function sendMicControlRecord(
session: unknown,
record: Uint8Array,
label: string,
rightTemple: boolean,
leftTemple: boolean,
): boolean {
return (session as any).sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
}
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
const communicator = session as any;
const proxy = new com.faceclaw.app.FaceclawMicStatusListener({
onMicStatus: (body: any, arm: string) => listener(toUint8Array(body), String(arm)),
});
communicator.addMicStatusListener(proxy);
return () => {
try {
communicator.removeMicStatusListener(proxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
};
}
+203
View File
@@ -0,0 +1,203 @@
import type { PebbleIconName } from "../graphics/pebble-icons";
/**
* Icons for notification sources that arrive without one: iPhone
* notifications over ANCS carry the app's bundle ID, its display name
* (fetched separately) and an ANCS category, but no icon. The icons are
* Pebble's (see scripts/generate-pebble-icons.cjs).
*
* The bundle IDs start from the Pebble firmware's own ANCS table (PebbleOS
* include/pbl/services/notifications/ancs/ancs_known_apps.h, minus retired
* apps), plus Apple's built-in apps and popular apps mapped to a generic
* pictogram. Every App Store bundle ID here was checked against Apple's
* iTunes lookup when it was added; Apple's built-in apps follow
* github.com/joeblau/apple-bundle-identifiers.
*
* Display names are a fallback for apps whose bundle ID varies per build:
* self-built apps such as Loop sign with their builder's team ID. They match
* the whole name, ignoring case, so generic words (an app called "United"
* or "Voice") are left out. Bundle prefixes catch regional variants
* (com.abbott.librelink.us, .gb, ...).
*
* Apple's Phone app is deliberately absent: its notifications are calls,
* missed calls and voicemail, which the ANCS category tells apart.
*/
type AppIcon = {
icon: PebbleIconName;
bundleIds: readonly string[];
bundlePrefixes?: readonly string[];
names?: readonly string[];
};
const APP_ICONS: readonly AppIcon[] = [
// Apple's built-in apps.
{ icon: "sms", bundleIds: ["com.apple.MobileSMS"] },
{ icon: "email", bundleIds: ["com.apple.mobilemail"] },
{ icon: "calendar", bundleIds: ["com.apple.mobilecal"] },
{ icon: "reminder", bundleIds: ["com.apple.reminders"] },
{ icon: "facetime", bundleIds: ["com.apple.facetime"] },
{ icon: "iosPhotos", bundleIds: ["com.apple.mobileslideshow"] },
{ icon: "alarmClock", bundleIds: ["com.apple.mobiletimer"] },
{ icon: "weather", bundleIds: ["com.apple.weather"] },
{ icon: "location", bundleIds: ["com.apple.findmy", "com.apple.Maps"] },
{ icon: "settings", bundleIds: ["com.apple.Preferences"] },
{ icon: "heart", bundleIds: ["com.apple.Health"] },
{ icon: "activity", bundleIds: ["com.apple.Fitness"] },
{ icon: "news", bundleIds: ["com.apple.news"] },
{ icon: "podcast", bundleIds: ["com.apple.podcasts"] },
{ icon: "music", bundleIds: ["com.apple.Music"] },
{ icon: "tv", bundleIds: ["com.apple.tv"] },
{ icon: "payBill", bundleIds: ["com.apple.Passbook"] },
{ icon: "stocks", bundleIds: ["com.apple.stocks"] },
{ icon: "appStore", bundleIds: ["com.apple.AppStore"] },
{ icon: "appleShortcuts", bundleIds: ["com.apple.shortcuts"] },
{ icon: "microphone", bundleIds: ["com.apple.VoiceMemos"] },
{ icon: "contacts", bundleIds: ["com.apple.MobileAddressBook"] },
{ icon: "shopping", bundleIds: ["com.apple.store.Jolly"] },
// Apps with an icon of their own.
{ icon: "airmail", bundleIds: ["com.airmailapp.iphone"], names: ["Airmail"] },
{ icon: "amazon", bundleIds: ["com.amazon.Amazon"], names: ["Amazon"] },
{ icon: "bbcNews", bundleIds: ["uk.co.bbc.news"], names: ["BBC News"] },
{ icon: "beeper", bundleIds: ["com.automattic.beeper"], names: ["Beeper"] },
{ icon: "bluesky", bundleIds: ["xyz.blueskyweb.app"], names: ["Bluesky"] },
{ icon: "discord", bundleIds: ["com.hammerandchisel.discord"], names: ["Discord"] },
{ icon: "duolingo", bundleIds: ["com.duolingo.DuolingoMobile"], names: ["Duolingo"] },
{ icon: "ebay", bundleIds: ["com.ebay.iphone"], names: ["eBay"] },
{ icon: "element", bundleIds: ["im.vector.app", "io.element.elementx"], names: ["Element", "Element X", "Element Classic"] },
{ icon: "facebook", bundleIds: ["com.facebook.Facebook"], names: ["Facebook"] },
{ icon: "gmail", bundleIds: ["com.google.Gmail"], names: ["Gmail"] },
{ icon: "googleChat", bundleIds: ["com.google.Dynamite"], names: ["Google Chat"] },
{ icon: "googleMaps", bundleIds: ["com.google.Maps"], names: ["Google Maps"] },
{ icon: "googlePhotos", bundleIds: ["com.google.photos"], names: ["Google Photos"] },
{ icon: "googleTasks", bundleIds: ["com.google.tasks"], names: ["Google Tasks"] },
{ icon: "homeAssistant", bundleIds: ["io.robbie.HomeAssistant"], names: ["Home Assistant"] },
{ icon: "instagram", bundleIds: ["com.burbn.instagram"], names: ["Instagram"] },
{ icon: "kakaoTalk", bundleIds: ["com.iwilab.KakaoTalk"], names: ["KakaoTalk"] },
{ icon: "kik", bundleIds: ["com.kik.chat"], names: ["Kik"] },
{ icon: "line", bundleIds: ["jp.naver.line"], names: ["LINE"] },
{ icon: "linkedIn", bundleIds: ["com.linkedin.LinkedIn"], names: ["LinkedIn"] },
{ icon: "messenger", bundleIds: ["com.facebook.Messenger"], names: ["Messenger"] },
{ icon: "omniFocus", bundleIds: ["com.omnigroup.OmniFocus4", "com.omnigroup.OmniFocus3.iOS"], names: ["OmniFocus"] },
{ icon: "outlook", bundleIds: ["com.microsoft.Office.Outlook"], names: ["Outlook"] },
{ icon: "reddit", bundleIds: ["com.reddit.Reddit"], names: ["Reddit"] },
{ icon: "signal", bundleIds: ["org.whispersystems.signal"], names: ["Signal"] },
{ icon: "simplex", bundleIds: ["chat.simplex.app"], names: ["SimpleX"] },
{ icon: "slack", bundleIds: ["com.tinyspeck.chatlyio"], names: ["Slack"] },
{ icon: "snapchat", bundleIds: ["com.toyopagroup.picaboo"], names: ["Snapchat"] },
{ icon: "steam", bundleIds: ["com.valvesoftware.Steam"], names: ["Steam"] },
{ icon: "substack", bundleIds: ["com.substack.Substack"], names: ["Substack"] },
{ icon: "swarm", bundleIds: ["com.foursquare.robin"], names: ["Swarm"] },
{ icon: "tapo", bundleIds: ["com.tplink.tapo"], names: ["Tapo"] },
{ icon: "teams", bundleIds: ["com.microsoft.skype.teams"], names: ["Teams", "Microsoft Teams"] },
{ icon: "telegram", bundleIds: ["ph.telegra.Telegraph"], names: ["Telegram"] },
{ icon: "threads", bundleIds: ["com.burbn.barcelona"], names: ["Threads"] },
{ icon: "tiktok", bundleIds: ["com.zhiliaoapp.musically"], names: ["TikTok"] },
{ icon: "twitch", bundleIds: ["tv.twitch"], names: ["Twitch"] },
// X, still under Twitter's bundle ID and drawn with its bird, as on Pebble.
{ icon: "twitter", bundleIds: ["com.atebits.Tweetie2"], names: ["X", "Twitter"] },
{ icon: "unifiProtect", bundleIds: ["com.ubnt.protect"], names: ["UniFi Protect"] },
{ icon: "viber", bundleIds: ["com.viber"], names: ["Viber"] },
{ icon: "waze", bundleIds: ["com.waze.iphone"], names: ["Waze"] },
{ icon: "weChat", bundleIds: ["com.tencent.xin"], names: ["WeChat"] },
{ icon: "whatsApp", bundleIds: ["net.whatsapp.WhatsApp"], names: ["WhatsApp"] },
{ icon: "whatsAppBusiness", bundleIds: ["net.whatsapp.WhatsAppSMB"], names: ["WA Business", "WhatsApp Business"] },
{ icon: "yahooMail", bundleIds: ["com.yahoo.Aerogram"], names: ["Yahoo Mail"] },
{ icon: "youtube", bundleIds: ["com.google.ios.youtube"], names: ["YouTube"] },
{ icon: "zoom", bundleIds: ["us.zoom.videomeetings"], names: ["Zoom"] },
// Apps drawn with a generic pictogram.
{ icon: "calendar", bundleIds: ["com.google.calendar", "com.flexibits.fantastical2.iphone"], names: ["Google Calendar", "Fantastical"] },
{ icon: "email", bundleIds: ["com.readdle.smartemail", "ch.protonmail.protonmail"], names: ["Proton Mail"] },
{ icon: "sms", bundleIds: ["com.google.GVDialer", "ch.threema.iapp", "com.groupme.iphone-app"], names: ["Threema", "GroupMe"] },
{ icon: "reminder", bundleIds: ["com.todoist.ios", "com.culturedcode.ThingsiPhone", "com.microsoft.to-do"], names: ["Todoist"] },
{ icon: "car", bundleIds: ["com.ubercab.UberClient", "com.zimride.instant"], names: ["Uber", "Lyft"] },
{ icon: "dinner", bundleIds: ["com.ubercab.UberEats", "doordash.DoorDashConsumer", "com.grubhub.search"], names: ["Uber Eats", "DoorDash", "Grubhub"] },
{
icon: "glucose",
bundleIds: [
"com.dexcom.G6", "com.dexcom.g7app", "com.dexcom.dexcomone", "com.dexcom.stelo", "com.dexcom.followglobal",
"org.nativescript.LibreLinkUp",
],
bundlePrefixes: ["com.abbott.librelink.", "com.abbott.libre3."],
names: ["Dexcom G6", "Dexcom G7", "Dexcom ONE", "LibreLink", "LibreLinkUp", "Loop", "Trio", "LoopFollow", "xDrip4iOS"],
},
{ icon: "music", bundleIds: ["com.spotify.client", "com.google.ios.youtubemusic", "com.pandora"], names: ["Spotify", "YouTube Music", "Pandora"] },
{ icon: "podcast", bundleIds: ["fm.overcast.overcast", "au.com.shiftyjelly.podcasts"], names: ["Overcast", "Pocket Casts"] },
{
icon: "tv",
bundleIds: ["com.netflix.Netflix", "com.hulu.plus", "com.disney.disneyplus", "com.amazon.aiv.AIVApp"],
names: ["Netflix", "Hulu", "Disney+", "Prime Video"],
},
{
icon: "payBill",
bundleIds: [
"net.kortina.labs.Venmo", "com.squareup.cash", "com.yourcompany.PPClient", "com.chase", "com.bankofamerica.BofA",
"com.wf.mobilebanking", "com.capitalone.enterprisemobilebanking",
"com.revolut.revolut", "com.transferwise.Transferwise", "de.no26.Number26", "com.bunq.ios",
],
names: ["Venmo", "Cash App", "PayPal", "Revolut", "N26", "bunq"],
},
{ icon: "stocks", bundleIds: ["com.robinhood.release.Robinhood", "com.vilcsak.bitcoin2"], names: ["Robinhood", "Coinbase"] },
{
icon: "flight",
bundleIds: [
"com.united.UnitedCustomerFacingIPhone", "com.delta.iphone.ver1", "com.aa.AmericanAirlines",
"com.southwest.iphoneprod", "com.flightyapp.flighty",
],
names: ["Fly Delta", "Flighty"],
},
{ icon: "weather", bundleIds: ["com.weather.TWC", "com.grailr.CARROTweather"] },
{ icon: "activity", bundleIds: ["com.strava.stravaride", "com.nike.nikeplus-gps", "com.fitbit.FitbitMobile"], names: ["Strava", "Fitbit"] },
{ icon: "heart", bundleIds: ["com.ouraring.oura"], names: ["Oura"] },
{ icon: "news", bundleIds: ["com.nytimes.NYTimes", "com.google.GoogleDigitalEditions"], names: ["Google News"] },
];
/**
* Icons for the ANCS categories (CategoryID: Other, IncomingCall,
* MissedCall, Voicemail, Social, Schedule, Email, News, HealthAndFitness,
* BusinessAndFinance, Location, Entertainment), for apps not listed above.
*/
const CATEGORY_ICONS: readonly PebbleIconName[] = [
"generic", "incomingCall", "missedCall", "voicemail", "sms", "calendar",
"email", "news", "heart", "stocks", "location", "tv",
];
type Index = { bundleIds: Map<string, PebbleIconName>; prefixes: [string, PebbleIconName][]; names: Map<string, PebbleIconName> };
let index: Index | null = null;
function buildIndex(): Index {
const built: Index = { bundleIds: new Map(), prefixes: [], names: new Map() };
for (const app of APP_ICONS) {
for (const id of app.bundleIds) built.bundleIds.set(id.toLowerCase(), app.icon);
for (const prefix of app.bundlePrefixes ?? []) built.prefixes.push([prefix.toLowerCase(), app.icon]);
for (const name of app.names ?? []) built.names.set(name.toLowerCase(), app.icon);
}
return built;
}
/**
* The icon for a notification's source app, by bundle ID (ignoring case;
* Apple treats bundle IDs case-insensitively), else by its display name;
* null when the app isn't known.
*/
export function notificationAppIcon(bundleId: string, appName: string): PebbleIconName | null {
index ??= buildIndex();
const id = bundleId.trim().toLowerCase();
if (id) {
const byId = index.bundleIds.get(id) ?? index.prefixes.find(([prefix]) => id.startsWith(prefix))?.[1];
if (byId) return byId;
}
const name = appName.trim().toLowerCase();
return (name && index.names.get(name)) || null;
}
/** The icon for an ANCS category (a CategoryID as a string); the bell for anything unknown. */
export function notificationCategoryIcon(category: string): PebbleIconName {
return CATEGORY_ICONS[Number(category)] ?? "generic";
}
/** Every icon the table refers to, for tests. */
export function allNotificationAppIcons(): PebbleIconName[] {
return Array.from(new Set(APP_ICONS.map((app) => app.icon).concat(CATEGORY_ICONS)));
}
+21 -24
View File
@@ -1,10 +1,11 @@
import { GrayImage } from '../graphics/image'
import { renderIcon, type IconName } from '../graphics/icons'
import type { GrayImage } from '../graphics/image'
import { pebbleIcon } from '../graphics/pebble-icons'
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
import { rememberNotificationSources } from './notification-sources'
import type { AndroidNotification } from './notification-types'
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons'
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
export const ALL_NOTIFICATIONS = 0x7fffffff
let active: AncsClient | null = null
@@ -32,36 +33,32 @@ export function onAndroidNotificationPosted(listener: (key: string) => void): ()
// (they hear every change), and an open popup closes on the repaint that follows.
export function onAndroidNotificationRemoved(_listener: (key: string) => void): () => void { return () => {} }
export function onAndroidNotificationsChanged(listener: () => void): () => void { return onAndroidNotificationPosted(listener) }
// ANCS does not provide app icons, so show one for the notification's ANCS
// category; never fetch notification sources or content from a third-party
// icon service. Indexed by CategoryID: Other, IncomingCall, MissedCall,
// Voicemail, Social, Schedule, Email, News, HealthAndFitness,
// BusinessAndFinance, Location, Entertainment.
const CATEGORY_ICONS: readonly IconName[] = ['bell','phone-incoming','phone-missed','voicemail','message-circle',
'calendar','mail','newspaper','heart-pulse','briefcase','map-pin','tv']
function categoryIcon(category: string): GrayImage {
return renderIcon(CATEGORY_ICONS[Number(category)] ?? 'bell',24)?.clone() ?? bell()
}
// Drawn fallback for when the SVG rasterizer fails.
function bell(): GrayImage {
const image = new GrayImage(24,24,0)
image.fillRoundedRect(6,5,12,13,220,5)
image.fillRect(4,17,16,2,220)
image.fillRect(10,20,4,2,220)
return image
// ANCS does not provide app icons, so show a bundled Pebble icon for the
// source app (by bundle ID or display name), else for the notification's
// ANCS category. Never fetch notification sources or content from a
// third-party icon service.
function notificationIcon(notification: AndroidNotification): GrayImage {
const name = notificationAppIcon(notification.packageName, notification.appName) ?? notificationCategoryIcon(notification.category)
return pebbleIcon(name).clone()
}
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
// One icon per source, standing for its first notification.
const sources = new Map<string, AndroidNotification>()
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
return {icons: shown.map(n => categoryIcon(n.category)),keys: shown.map(n => n.key),stale:false}
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
}
export function readNotificationIconByKey(key: string, _allowStale: boolean) {
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key)
return {icon: notification ? categoryIcon(notification.category) : null,stale:false}
return {icon: notification ? notificationIcon(notification) : null,stale:false}
}
// ANCS carries no attachments.
// ANCS carries no attachments, avatars or custom layouts.
export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
return {picture: null,stale:false}
return {image: null,stale:false}
}
export function readNotificationLargeIcon(_notification: AndroidNotification, _size: number, _allowStale: boolean) {
return {image: null,stale:false}
}
export function readNotificationCustomView(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
return {image: null,stale:false}
}
+121 -44
View File
@@ -3,11 +3,12 @@ import { logCurrent, spanCurrent } from "./frame-timings";
import { toUint8Array } from "../util/array-util";
import { rememberNotificationSources } from "./notification-sources";
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
import { NOTIFICATION_ICON_SIZE } from "./notification-types";
declare const com: any;
declare const java: any;
const ICON_SIZE = 24;
const ICON_SIZE = NOTIFICATION_ICON_SIZE;
/** Ask the native listener for every active source, including those below the list's display limit. */
export const ALL_NOTIFICATIONS = 0x7fffffff;
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
@@ -23,11 +24,6 @@ let cachedKeys: string[] = [];
let cachedAtMs = 0;
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
const KEYED_ICON_CACHE_MAX = 128;
/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */
const pictureCache = new Map<string, {
postTime: number; maxWidth: number; maxHeight: number; picture: GrayImage | null; atMs: number;
}>();
const PICTURE_CACHE_MAX = 8;
let notificationListenerProxy: any = null;
const notificationPostedListeners = new Set<(notificationKey: string) => void>();
const notificationRemovedListeners = new Set<(notificationKey: string) => void>();
@@ -159,48 +155,86 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not
return { icon: icon?.clone() ?? null, stale: false };
}
export type NotificationPictureResult = {
export type NotificationImageResult = {
/** Shared with the cache: blit it, don't draw on it. */
picture: GrayImage | null;
image: GrayImage | null;
/** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */
stale: boolean;
};
/**
* Images belonging to one notification (its picture, large icon or custom
* layout), fetched to fit a box, under the allow-stale contract of
* readNotificationIconByKey. Cached per post of the notification: an ongoing
* notification updated in place (Dexcom redraws its graph and reading every
* few minutes) fires no posted event, but its postTime moves. ICON_CACHE_MS
* is the backstop here too. Under allowStale an earlier fetch stands in, if
* it fits.
*/
class NotificationImageCache {
private readonly entries = new Map<string, {
postTime: number; maxWidth: number; maxHeight: number; image: GrayImage | null; atMs: number;
}>();
constructor(private readonly maxEntries: number, private readonly span: string) {}
read(
notification: AndroidNotification,
maxWidth: number,
maxHeight: number,
allowStale: boolean,
fetch: (width: number, height: number) => GrayImage | null,
): NotificationImageResult {
const width = Math.round(maxWidth);
const height = Math.round(maxHeight);
if (!global.isAndroid || width <= 0 || height <= 0) return { image: null, stale: false };
const now = Date.now();
const cached = this.entries.get(notification.key);
if (
cached && cached.postTime === notification.postTime && cached.maxWidth === width
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
) {
return { image: cached.image, stale: false };
}
if (allowStale) {
const fits = cached?.image && cached.image.width <= width && cached.image.height <= height;
return { image: fits ? cached!.image : null, stale: true };
}
const image = spanCurrent(this.span, () => fetch(width, height));
this.entries.delete(notification.key);
this.entries.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, image, atMs: now });
while (this.entries.size > this.maxEntries) {
const oldestKey = this.entries.keys().next().value;
if (oldestKey === undefined) break;
this.entries.delete(oldestKey);
}
return { image, stale: false };
}
}
/** Pictures and custom layouts are the size of a detail view, not icons: keep just the recently viewed few. */
const pictureCache = new NotificationImageCache(8, "fetch-notification-picture");
const customViewCache = new NotificationImageCache(8, "fetch-notification-custom-view");
/** Large icons show on every list card drawn, so keep a screenful and then some. */
const largeIconCache = new NotificationImageCache(32, "fetch-notification-large-icon");
/** Avatars and logos: photographic tone, but undithered, which at icon size reads as noise. */
const LARGE_ICON_TONE = { gamma: PHOTO_TONE.gamma, dither: false };
/**
* The picture attached to a notification (hasPicture), scaled to fit
* maxWidth x maxHeight with the photo tone and inverted if it is drawn on a
* light background (GrayPacket.hasLightBackground), under the allow-stale contract of
* readNotificationIconByKey. Cached per post of the notification: an ongoing
* notification updated in place (Dexcom redraws its graph every few minutes)
* fires no posted event, but its postTime moves. ICON_CACHE_MS is the
* backstop here too. Under allowStale an earlier fetch stands in, if it fits.
* light background (GrayPacket.hasLightBackground); see NotificationImageCache.
*/
export function readNotificationPicture(
notification: AndroidNotification,
maxWidth: number,
maxHeight: number,
allowStale: boolean,
): NotificationPictureResult {
const width = Math.round(maxWidth);
const height = Math.round(maxHeight);
if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) {
return { picture: null, stale: false };
}
const now = Date.now();
const cached = pictureCache.get(notification.key);
if (
cached && cached.postTime === notification.postTime && cached.maxWidth === width
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
) {
return { picture: cached.picture, stale: false };
}
if (allowStale) {
const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height;
return { picture: fits ? cached!.picture : null, stale: true };
}
const picture = spanCurrent("fetch-notification-picture", () =>
): NotificationImageResult {
if (!notification.hasPicture) return { image: null, stale: false };
return pictureCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
grayImageFromPacket(
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray(
notification.key,
@@ -212,16 +246,56 @@ export function readNotificationPicture(
// lit panel on the glasses: show it light-on-dark.
true,
),
),
);
pictureCache.delete(notification.key);
pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now });
while (pictureCache.size > PICTURE_CACHE_MAX) {
const oldestKey = pictureCache.keys().next().value;
if (oldestKey === undefined) break;
pictureCache.delete(oldestKey);
}
return { picture, stale: false };
));
}
/**
* A notification's large icon (hasLargeIcon: a sender's avatar, a site's
* logo), scaled to fit size x size, a light background inverted as for
* pictures; see NotificationImageCache.
*/
export function readNotificationLargeIcon(
notification: AndroidNotification,
size: number,
allowStale: boolean,
): NotificationImageResult {
if (!notification.hasLargeIcon) return { image: null, stale: false };
return largeIconCache.read(notification, size, size, allowStale, (width) =>
grayImageFromPacket(
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationLargeIconGray(
notification.key,
width,
LARGE_ICON_TONE.gamma,
LARGE_ICON_TONE.dither,
),
));
}
/**
* A notification's own layout (hasCustomView) drawn as the phone's shade
* draws it, at most a pixel per dp and trimmed to what it draws, so usually
* smaller than maxWidth x maxHeight; dark-on-light layouts come out
* light-on-dark. See NotificationCustomViews.renderGray and
* NotificationImageCache.
*/
export function readNotificationCustomView(
notification: AndroidNotification,
maxWidth: number,
maxHeight: number,
allowStale: boolean,
): NotificationImageResult {
if (!notification.hasCustomView) return { image: null, stale: false };
return customViewCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
grayImageFromPacket(
// Flat UI tone: it is mostly text and icons.
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationCustomViewGray(
notification.key,
width,
height,
1,
false,
),
));
}
export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] {
@@ -354,6 +428,9 @@ function normalizeNotification(value: any): AndroidNotification | null {
actions,
foregroundService: Boolean(value.foregroundService),
hasPicture: Boolean(value.hasPicture),
hasLargeIcon: Boolean(value.hasLargeIcon),
hasCustomView: Boolean(value.hasCustomView),
customText: Array.isArray(value.customText) ? value.customText.map((text: unknown) => String(text)).filter(Boolean) : [],
};
}
+18
View File
@@ -1,4 +1,13 @@
/** Shared notification model; Android names are retained for existing callers. */
/**
* Side of a notification's (square) source icon wherever one is drawn: the
* status bar, notification cards and views, the Glanceboard. 25 is the size
* of the Pebble icons iPhone notifications use (app/graphics/pebble-icons.ts),
* which are drawn on that pixel grid; Android's icons render at any size.
*/
export const NOTIFICATION_ICON_SIZE = 25;
export type AndroidNotificationAction = { index: number; title: string; enabled: boolean };
export type AndroidNotification = {
key: string; packageName: string; appName: string; title: string; text: string;
@@ -9,6 +18,15 @@ export type AndroidNotification = {
foregroundService?: boolean;
/** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */
hasPicture?: boolean;
/** Carries a large icon (Android: a sender's avatar or logo), fetched separately by readNotificationLargeIcon. */
hasLargeIcon?: boolean;
/** Has its own layout (Android RemoteViews), drawn by readNotificationCustomView. */
hasCustomView?: boolean;
/**
* The custom layout's text, one entry per text view, when the notification has no title or
* text of its own (Dexcom's glucose reading and units).
*/
customText?: string[];
/** Full message size reported by iOS, even when the fetched text is bounded. */
messageSize?: number;
};
+159
View File
@@ -0,0 +1,159 @@
import { fetchTextWithUserAgent, type TextResponse } from "../util/http";
import { normalizeOpenAiCompatibleUrl } from "./openai-chat";
export type DiscoveredCompatibleModel = {
/** Exact identifier to send as model in Chat Completions. */
id: string;
/** Human-friendly catalog name, or id when the server supplies no name. */
name: string;
/** true/false when advertised; null means the server did not say. */
supportsTools: boolean | null;
offline: boolean;
deprecated: boolean;
beta: boolean;
};
type DiscoveryOptions = {
baseUrl: string;
apiKey: string;
/** Bypass the short-lived successful-response cache. */
forceRefresh?: boolean;
};
type DiscoveryResponse = Pick<TextResponse, "status" | "text">;
type DiscoveryHttp = (url: string, init: {
method: string;
headers: Record<string, string>;
timeoutMs: number;
dontFollowRedirects: boolean;
}) => Promise<DiscoveryResponse>;
type DiscoveryDependencies = {
http?: DiscoveryHttp;
now?: () => number;
};
type CacheEntry = {
apiKey: string;
fetchedAt: number;
models: DiscoveredCompatibleModel[];
};
export const COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS = 10_000;
const CACHE_TTL_MS = 60_000;
const MAX_MODELS = 500;
const MAX_DISCOVERY_BODY_CHARS = 2_000_000;
const cache = new Map<string, CacheEntry>();
/**
* Discover models from the configured server's own /models endpoint. No URL
* probing, inference calls, disk persistence, or automatic model selection.
*/
export async function discoverOpenAiCompatibleModels(
options: DiscoveryOptions,
dependencies: DiscoveryDependencies = {},
): Promise<DiscoveredCompatibleModel[]> {
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
if (!baseUrl) throw new Error("Set a valid compatible API URL first");
const apiKey = options.apiKey.trim();
const now = dependencies.now ?? Date.now;
const cached = cache.get(baseUrl);
if (!options.forceRefresh && cached && cached.apiKey === apiKey && now() - cached.fetchedAt < CACHE_TTL_MS) {
return cached.models;
}
const http = dependencies.http ?? defaultHttp;
const headers: Record<string, string> = {};
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
let response: DiscoveryResponse;
try {
response = await http(`${baseUrl}/models`, {
method: "GET",
headers,
timeoutMs: COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS,
// Do not forward a credential to a redirect target.
dontFollowRedirects: true,
});
} catch {
throw new Error("Could not reach the configured server");
}
if (response.status < 200 || response.status >= 300) {
throw new Error(describeHttpError(response.status));
}
let body: any;
let rawBody: string;
try {
rawBody = await response.text();
if (rawBody.length > MAX_DISCOVERY_BODY_CHARS) throw new Error("too large");
body = JSON.parse(rawBody);
} catch {
throw new Error("The server returned an invalid model list");
}
if (!body || !Array.isArray(body.data)) {
throw new Error("The server did not return an OpenAI-compatible model list");
}
const models = parseModels(body.data);
cache.set(baseUrl, { apiKey, fetchedAt: now(), models });
return models;
}
/** Parse OpenAI data[] rows, retaining richer Venice metadata when present. */
export function parseDiscoveredCompatibleModels(rows: any[]): DiscoveredCompatibleModel[] {
return parseModels(rows);
}
function parseModels(rows: any[]): DiscoveredCompatibleModel[] {
const byId = new Map<string, DiscoveredCompatibleModel>();
for (const row of rows) {
if (!row || typeof row.id !== "string" || !row.id.trim()) continue;
const id = row.id.trim();
if (byId.has(id)) continue;
const spec = row.model_spec && typeof row.model_spec === "object" ? row.model_spec : null;
const capabilities = spec?.capabilities && typeof spec.capabilities === "object" ? spec.capabilities : null;
const catalogName = typeof spec?.name === "string" ? spec.name.trim() : "";
const genericName = typeof row.name === "string" ? row.name.trim() : "";
byId.set(id, {
id,
name: catalogName || genericName || id,
supportsTools: typeof capabilities?.supportsFunctionCalling === "boolean"
? capabilities.supportsFunctionCalling
: null,
offline: spec?.offline === true,
deprecated: Boolean(spec?.deprecation),
beta: spec?.betaModel === true || spec?.beta === true,
});
if (byId.size >= MAX_MODELS) break;
}
return Array.from(byId.values()).sort((a, b) => {
if (a.offline !== b.offline) return a.offline ? 1 : -1;
const aTools = a.supportsTools === true ? 0 : a.supportsTools === null ? 1 : 2;
const bTools = b.supportsTools === true ? 0 : b.supportsTools === null ? 1 : 2;
if (aTools !== bTools) return aTools - bTools;
return (a.name || a.id).localeCompare(b.name || b.id) || a.id.localeCompare(b.id);
});
}
async function defaultHttp(url: string, init: {
method: string;
headers: Record<string, string>;
timeoutMs: number;
dontFollowRedirects: boolean;
}): Promise<DiscoveryResponse> {
return fetchTextWithUserAgent(url, init);
}
function describeHttpError(status: number): string {
if (status === 401) return "The server rejected the API key (HTTP 401)";
if (status === 403) return "The API key cannot list these models (HTTP 403)";
if (status === 404) return "Model discovery is not supported at this URL (HTTP 404)";
if (status === 429) return "The server is rate limiting requests (HTTP 429)";
if (status >= 500) return `The server is unavailable (HTTP ${status})`;
if (status >= 300 && status < 400) return `The server redirected the request; use its direct API URL (HTTP ${status})`;
return `Model discovery failed (HTTP ${status})`;
}
/** Test hook; does not affect any saved setting. */
export function clearCompatibleModelDiscoveryCache(): void {
cache.clear();
}
+330
View File
@@ -0,0 +1,330 @@
import type {
LlmContentBlock,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { openSseRequest, type SseListener } from "./sse";
/** True only for Venice's official API root, never for lookalike hosts. */
export function isVeniceApiBaseUrl(input: string): boolean {
return normalizeOpenAiCompatibleUrl(input) === "https://api.venice.ai/api/v1";
}
/** Streaming client for OpenAI-compatible Chat Completions APIs. */
export type OpenAiCompatibleStreamOptions = LlmStreamOptions & {
/** Explicit API root, for example https://api.venice.ai/api/v1 or https://host/api. */
baseUrl: string;
/** Venice-only web search; never sent to other compatible providers. */
webSearch?: boolean;
};
type PendingToolCall = {
id: string;
name: string;
inputJson: string;
};
/** Remove an optional endpoint suffix, but never guess a provider's /v1 or /api prefix. */
export function normalizeOpenAiCompatibleUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
url = url.replace(/\/chat\/completions$/i, "").replace(/\/+$/, "");
return /^https?:\/\/[^/\s?#@\\]+(?:\/[^\s?#\\]*)?$/i.test(url) ? url : "";
}
export function streamOpenAiCompatibleChat(options: OpenAiCompatibleStreamOptions): LlmStreamHandle {
const apiKey = options.apiKey.trim();
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
let request: { cancel(): void } | null = null;
let settled = false;
let text = "";
let finishReason: string | null = null;
const toolCalls = new Map<number, PendingToolCall>();
const providerMetadata: Record<string, any> = {};
const reasoningDetails = new ReasoningDetailsAccumulator();
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
const hasToolCalls = toolCalls.size > 0;
// Never execute calls from a truncated stream or a length-limited response.
if (hasToolCalls && finishReason !== "tool_calls" && finishReason !== "stop") {
fail("OpenAI-compatible stream ended before its tool calls completed");
return;
}
if (!text && !hasToolCalls) {
fail("OpenAI-compatible provider ended the turn without a reply");
return;
}
const content: LlmContentBlock[] = [];
if (text) content.push({ type: "text", text });
const details = reasoningDetails.entries();
if (details.length) providerMetadata.reasoning_details = details;
if (Object.keys(providerMetadata).length) {
content.push({ type: "provider_item", provider: "openai-compatible", item: providerMetadata });
}
for (const index of Array.from(toolCalls.keys()).sort((a, b) => a - b)) {
const call = toolCalls.get(index)!;
if (!call.name) {
fail("OpenAI-compatible provider returned a tool call without a name");
return;
}
let input: any = {};
if (call.inputJson.trim()) {
try {
input = JSON.parse(call.inputJson);
} catch {
fail("OpenAI-compatible provider returned malformed tool arguments");
return;
}
}
content.push({
type: "tool_use",
id: call.id || `call_${index + 1}`,
name: call.name,
input,
});
}
settled = true;
request?.cancel();
options.onDone({
text,
content,
stopReason: finishReason === "content_filter" ? "refusal"
: hasToolCalls ? "tool_use"
: finishReason === "length" ? "max_tokens" : "end_turn",
});
};
if (!baseUrl || !options.model.trim()) {
setTimeout(() => fail(!baseUrl
? "Set a valid OpenAI-compatible API URL"
: "Set an OpenAI-compatible model ID"), 0);
return { cancel: () => { settled = true; } };
}
const body: any = {
model: options.model.trim(),
messages: buildChatMessages(options),
stream: true,
};
// Let the provider choose its default unless a one-shot feature supplies a cap.
if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens;
if (options.tools?.length) {
body.tools = options.tools.map((tool) => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
},
}));
}
// Provider-specific parameters must never leak to another endpoint, even
// when a saved Venice preference is still enabled.
if (options.webSearch === true && isVeniceApiBaseUrl(baseUrl)) {
body.venice_parameters = { enable_web_search: "on" };
}
// Do not send provider-specific reasoning fields by default. Compatible
// servers vary widely; advanced reasoning controls can be added separately.
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
const listener: SseListener = {
onLine: (line: string) => {
if (settled || !line.startsWith("data:")) return;
const payload = line.slice(5).trim();
if (!payload) return;
if (payload === "[DONE]") {
if (finishReason || (!toolCalls.size && text)) done();
else fail("OpenAI-compatible stream ended before the response completed");
return;
}
let event: any;
try {
event = JSON.parse(payload);
} catch {
return;
}
if (event?.error) {
fail(`OpenAI-compatible: ${String(event.error.message ?? event.error)}`);
return;
}
const choice = event?.choices?.[0];
if (!choice) return; // Usage and provider-metadata chunks may have no choices.
const delta = choice.delta ?? {};
if (typeof delta.content === "string" && delta.content) {
text += delta.content;
options.onTextDelta?.(delta.content, text);
}
// Reasoning is not displayed. Preserve provider metadata for replay in
// multi-turn tool loops when the service supplies it.
if (Array.isArray(delta.reasoning_details)) reasoningDetails.add(delta.reasoning_details);
if (delta.thought_signature !== undefined) providerMetadata.thought_signature = delta.thought_signature;
if (Array.isArray(delta.tool_calls)) {
delta.tool_calls.forEach((call: any, position: number) => {
const index = Number.isInteger(call?.index) ? Number(call.index) : position;
const pending = toolCalls.get(index) ?? { id: "", name: "", inputJson: "" };
if (call?.id) pending.id = String(call.id);
if (call?.function?.name) pending.name = String(call.function.name);
if (typeof call?.function?.arguments === "string") pending.inputJson += call.function.arguments;
toolCalls.set(index, pending);
});
}
if (choice.finish_reason) finishReason = String(choice.finish_reason);
},
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
onComplete: () => {
// Some compatible servers close after the finish chunk without [DONE].
if (finishReason) done();
else fail("OpenAI-compatible stream ended before the response completed");
},
onFailure: (message: string) => fail(`OpenAI-compatible connection failed: ${String(message)}`),
};
try {
request = openSseRequest(`${baseUrl}/chat/completions`, JSON.stringify(body), headers, listener);
} catch (error) {
setTimeout(() => fail(`OpenAI-compatible request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => { settled = true; } };
}
return {
cancel: () => {
settled = true;
try { request?.cancel(); } catch { /* ignore */ }
},
};
}
/** String fields of a reasoning_details entry that stream in pieces. */
const REASONING_DETAIL_TEXT_FIELDS = ["text", "summary", "data"];
/**
* Rebuilds the complete reasoning_details array (OpenRouter's format) from
* streamed deltas. Each delta carries fragments; fragments with the same
* `index` belong to one entry, so their text, summary, and data concatenate
* and their other fields (e.g. a signature that arrives after the text) fill
* in. Entries without an index are kept as sent. The replayed array must
* match what the model produced, so it is reassembled whole rather than
* replaying only the latest chunk.
*/
class ReasoningDetailsAccumulator {
private readonly ordered: Record<string, any>[] = [];
private readonly byIndex = new Map<number, Record<string, any>>();
add(fragments: any[]): void {
for (const fragment of fragments) {
if (!fragment || typeof fragment !== "object" || Array.isArray(fragment)) continue;
const index = Number.isInteger(fragment.index) ? Number(fragment.index) : null;
const existing = index === null ? undefined : this.byIndex.get(index);
if (!existing) {
const entry = { ...fragment };
this.ordered.push(entry);
if (index !== null) this.byIndex.set(index, entry);
continue;
}
for (const [key, value] of Object.entries(fragment)) {
if (value === undefined || value === null) continue;
if (REASONING_DETAIL_TEXT_FIELDS.includes(key) && typeof value === "string" && typeof existing[key] === "string") {
existing[key] += value;
} else if (existing[key] === undefined || existing[key] === null) {
existing[key] = value;
}
}
}
}
entries(): Record<string, any>[] {
return this.ordered;
}
}
/** Convert neutral conversation history to Chat Completions messages. */
function buildChatMessages(options: OpenAiCompatibleStreamOptions): any[] {
const messages: any[] = [];
if (options.system) messages.push({ role: "system", content: options.system });
for (const message of options.messages) {
if (typeof message.content === "string") {
messages.push({ role: message.role, content: message.content });
continue;
}
if (message.role === "assistant") {
let text = "";
const toolCalls: any[] = [];
let providerMetadata: Record<string, any> = {};
for (const block of message.content) {
if (block.type === "text") text += block.text;
else if (block.type === "tool_use") {
toolCalls.push({
id: block.id,
type: "function",
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
});
} else if (block.type === "provider_item" && block.provider === "openai-compatible") {
providerMetadata = { ...providerMetadata, ...block.item };
}
}
if (text || toolCalls.length) {
messages.push({
role: "assistant",
content: text || (toolCalls.length ? null : ""),
...(toolCalls.length ? { tool_calls: toolCalls } : {}),
...providerMetadata,
});
}
continue;
}
let pendingText = "";
const flushText = () => {
if (!pendingText) return;
messages.push({ role: "user", content: pendingText });
pendingText = "";
};
for (const block of message.content) {
if (block.type === "text") pendingText += block.text;
else if (block.type === "tool_result") {
flushText();
messages.push({
role: "tool",
tool_call_id: block.tool_use_id,
content: block.is_error ? `Error: ${block.content}` : block.content,
});
}
}
flushText();
}
return messages;
}
function describeHttpError(code: number, body: string): string {
let detail = "";
try {
const parsed = JSON.parse(body);
detail = String(parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? "");
} catch {
// Non-JSON error body.
}
switch (code) {
case 401: return "OpenAI-compatible: invalid API key";
case 403: return "OpenAI-compatible: API key lacks permission";
case 429: return "OpenAI-compatible: rate limited, try again shortly";
case 500:
case 502:
case 503:
case 504:
return "OpenAI-compatible: service unavailable, try again shortly";
default: return `OpenAI-compatible: HTTP ${code}${detail ? ` (${detail})` : ""}`;
}
}
+3 -3
View File
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
function sessionConfig(): object {
function sessionConfig(language: string): object {
return {
type: "session.update",
session: {
@@ -209,7 +209,7 @@ function sessionConfig(): object {
audio: {
input: {
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
transcription: { model: MODEL_ID },
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
// Push-to-talk owns the utterance boundary; no server VAD.
turn_detection: null,
},
+22
View File
@@ -0,0 +1,22 @@
/** Platform-neutral process resource sample, shared by resource-usage.ts and resource-usage.ios.ts. */
export type ResourceSample = {
/** Monotonic clock (counting deep sleep), ms. */
atMs: number;
/** Cumulative CPU time of every thread in the process, ms. */
cpuMs: number;
threads: number;
/** One value per entry of the platform's MEMORY_SERIES, MiB. */
memoryMb: number[];
/** Platform-specific details for the latest sample (peaks, limits, thermal state). */
status: string;
};
export function kbToMb(kb: number): number {
return Number.isFinite(kb) ? Math.max(0, kb) / 1024 : 0;
}
export function formatMb(value: number): string {
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
return `${Math.round(value)}M`;
}
+37
View File
@@ -0,0 +1,37 @@
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
export * from "./resource-usage-types";
declare const FaceclawResourceUsage: any;
/**
* Memory series charted by the resource monitor, headline first: the physical
* footprint (what jetsam enforces its limit against, and what Xcode's memory
* gauge shows), resident memory, and bytes in use across the malloc zones.
*/
export const MEMORY_SERIES: readonly string[] = ["footprint", "resident", "malloc"];
const THERMAL_STATES = ["nominal", "fair", "serious", "critical"];
export function sampleResourceUsage(): ResourceSample {
const raw = JSON.parse(String(FaceclawResourceUsage.sample())) as Record<string, number | boolean | undefined>;
const kb = (key: string) => kbToMb(Number(raw[key] ?? 0));
const footprintMb = kb("footprintKb");
const peakMb = kb("footprintPeakKb");
const availableMb = kb("availableKb");
const thermal = THERMAL_STATES[Number(raw.thermalState)];
return {
atMs: Number(raw.atMs),
cpuMs: Number(raw.cpuMs ?? 0),
threads: Number(raw.threads ?? 0),
memoryMb: [footprintMb, kb("residentKb"), kb("mallocKb")],
status: [
// Footprint plus the remaining headroom is the jetsam limit. No headroom
// means no limit applies (the simulator) or it was just exceeded.
availableMb > 0 ? `limit ${formatMb(footprintMb + availableMb)}` : "",
peakMb > 0 ? `peak ${formatMb(peakMb)}` : "",
thermal ? `thermal ${thermal}` : "",
raw.lowPower === true ? "Low Power Mode" : "",
].filter(Boolean).join(" · "),
};
}
+28
View File
@@ -0,0 +1,28 @@
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
export * from "./resource-usage-types";
declare const com: any;
/**
* Memory series charted by the resource monitor, headline first: RSS from
* /proc/self/status (what Android records at exit), the native malloc heap,
* and the used Java heap.
*/
export const MEMORY_SERIES: readonly string[] = ["RSS", "native", "Java"];
export function sampleResourceUsage(): ResourceSample {
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
const peakRssMb = kbToMb(Number(raw[7]));
const javaCommittedMb = kbToMb(Number(raw[5]));
return {
atMs: Number(raw[0]),
cpuMs: Number(raw[1]),
memoryMb: [kbToMb(Number(raw[2])), kbToMb(Number(raw[3])), kbToMb(Number(raw[4]))],
threads: Number(raw[6]),
status: [
peakRssMb > 0 ? `peak RSS ${formatMb(peakRssMb)}` : "",
`Java heap ${formatMb(javaCommittedMb)} committed`,
].filter(Boolean).join(" · "),
};
}
+2
View File
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
// A hint, not a restriction: other languages are still recognized.
...(this.options.language ? { language_hints: [this.options.language] } : {}),
}),
);
if (this.closed) return;
+193
View File
@@ -0,0 +1,193 @@
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
declare const FaceclawSpeechAnalyzer: any;
/**
* How long finish() waits for the last results: the model only has to settle
* the few seconds of audio it hadn't finalized yet.
*/
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
/**
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
* whose last word ends within this much before that (the model's word ends
* against the detector's level threshold) leaves nothing before the pause
* unfinalized, so its paragraph is complete.
*/
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
type AnalyzerEvent = {
kind: string;
message?: string;
locale?: string;
text?: string;
final?: boolean;
start?: number | null;
end?: number | null;
speechEnd?: number | null;
};
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
export function speechAnalyzerSupported(): boolean {
try {
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
} catch {
return false;
}
}
/**
* On-device long-form transcription for continuous capture, wrapped like a
* cloud provider: it buffers audio until the model is ready (downloading it on
* first use) and gets pause commits.
*/
export function createSpeechAnalyzerClient(
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): ReconnectingSttClient {
return new ReconnectingSttClient(
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
continuous,
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
);
}
/**
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
* The model finalizes text in its own pieces, a few seconds behind, and can't
* be made to settle at a pause; this gathers its pieces into paragraphs ended
* by pauses, like the cloud providers' segments. A paragraph is shown as
* provisional text and sent as one final (paragraphBreakAfter) once a final
* reaches its pause, and the last one as a plain final when the audio ends.
*
* Only finals place breaks, by when their last word ends (the model's
* word-end times are accurate; first words' starts reach back into the
* silence before them, and volatile results only carry the range of all audio
* not finalized yet).
*/
export class SpeechAnalyzerSttClient implements CloudSttClient {
private native: any = null;
private samples = 0;
private finishing = false;
// Audio times (s) of pauses whose paragraph hasn't ended yet.
private pauses: number[] = [];
// Finalized text since the last paragraph break, when its last word ends,
// and the model's provisional rest.
private paragraph: string[] = [];
private paragraphEnd: number | null = null;
private volatile = "";
constructor(private readonly options: CloudSttOptions) {}
start(): void {
if (this.native) return;
const native = FaceclawSpeechAnalyzer.new();
native.eventHandler = (json: string) => {
if (this.native === native) this.receive(JSON.parse(json));
};
this.native = native;
native.startWithLanguage(this.options.language ?? "");
}
acceptPcm(pcm: Uint8Array): void {
if (!this.native || this.finishing || !pcm.length) return;
this.samples += pcm.length >> 1;
const copy = pcm.slice();
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
}
/** A pause: it ends the paragraph once the words before it are final. */
commitSegment(): void {
if (!this.native || this.finishing) return;
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
finish(): void {
if (!this.native || this.finishing) return;
this.finishing = true;
this.native.finish();
}
stop(): void {
const native = this.native;
this.native = null;
native?.cancel();
}
private receive(event: AnalyzerEvent): void {
switch (event.kind) {
case "status":
this.options.onStatus(event.message ?? "");
break;
case "ready":
this.options.onStatus("Listening (on-device)...");
this.options.onReady?.();
break;
case "transcript":
this.transcript(event);
break;
case "finished": {
const text = this.liveText();
this.paragraph = [];
this.volatile = "";
this.stop();
this.options.onTranscript({ text, isFinal: true });
break;
}
case "error":
this.stop();
this.options.onError(event.message ?? "On-device transcription failed.");
break;
}
}
private transcript(event: AnalyzerEvent): void {
const text = (event.text ?? "").trim();
let ended = false;
if (!event.final) {
this.volatile = text;
} else {
this.volatile = "";
if (text) {
const end = event.speechEnd ?? event.end ?? null;
// Words ending after a pause come after it. (One whose words run
// across it, if the model didn't split there, breaks a little early.)
if (end !== null && this.pauses.some((pause) => end > pause)) {
this.pauses = this.pauses.filter((pause) => pause >= end);
this.endParagraph();
}
this.paragraph.push(text);
this.paragraphEnd = end;
ended = this.endCompleteParagraph();
}
}
// The paragraph's final already cleared the provisional text.
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
/** End the paragraph if its last word ends just before a pause. */
private endCompleteParagraph(): boolean {
const end = this.paragraphEnd;
if (!this.paragraph.length || end === null) return false;
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
if (!this.pauses.some(reached)) return false;
this.pauses = this.pauses.filter((pause) => !reached(pause));
this.endParagraph();
return true;
}
private endParagraph(): void {
const text = this.paragraph.join(" ");
this.paragraph = [];
this.paragraphEnd = null;
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
}
private liveText(): string {
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
}
}
+273
View File
@@ -0,0 +1,273 @@
import { getStringSetting, setStringSetting } from "./settings-store";
import { buildQuery, SpotifyApiError, SpotifyClient, type SpotifyFetch, type SpotifyTokenSource } from "./spotify-web-api";
/**
* Spotify sign-in: OAuth authorization code with PKCE against the user's own
* client ID (no client secret), redirected to a loopback address that the
* phone catches (FaceclawWebSignIn on iOS). Spotify accepts http only for
* loopback IP literals, and custom-scheme redirects fail with "insecure
* redirect URI" for apps created since 2025, so the redirect is
* http://127.0.0.1 on a fixed port the user registers in the dashboard.
*
* The tokens are stored with the client ID they were issued to; changing the
* client ID setting signs out.
*/
export const SPOTIFY_ACCOUNTS_BASE = "https://accounts.spotify.com";
export const SPOTIFY_REDIRECT_PORT = 43823;
export const SPOTIFY_REDIRECT_PATH = "/spotify-callback";
export const SPOTIFY_REDIRECT_URI = `http://127.0.0.1:${SPOTIFY_REDIRECT_PORT}${SPOTIFY_REDIRECT_PATH}`;
export const SPOTIFY_DASHBOARD_URL = "https://developer.spotify.com/dashboard";
export const SPOTIFY_CLIENT_ID_KEY = "music.spotify.clientId";
export const SPOTIFY_ACCOUNT_KEY = "music.spotify.account";
export const SPOTIFY_SCOPES = [
"user-read-playback-state",
"user-modify-playback-state",
"user-read-currently-playing",
"user-read-recently-played",
"playlist-read-private",
"playlist-read-collaborative",
"user-library-read",
"user-library-modify",
"user-follow-read",
];
/** Refresh this long before Spotify's stated expiry (an hour), so a token never lapses mid-request. */
const EXPIRY_MARGIN_MS = 60_000;
export type SpotifyAccount = {
clientId: string;
accessToken: string;
refreshToken: string;
/** Epoch ms. */
expiresAtMs: number;
userId: string;
displayName: string;
/** "premium", "free", ...; playback control needs premium. */
product: string;
};
export type SpotifyTokens = { accessToken: string; refreshToken: string; expiresAtMs: number };
/** The phone side of a sign-in: PKCE material and the browser sheet that catches the loopback redirect. */
export type WebSignIn = {
/** A fresh PKCE verifier and its S256 challenge. */
createPkce(): { verifier: string; challenge: string };
randomState(): string;
/** Show the provider's page; resolves to the redirect URL, rejects with "cancelled" or why it failed. */
open(authorizeUrl: string, port: number, path: string): Promise<string>;
cancel(): void;
};
export function spotifyClientId(): string {
return getStringSetting(SPOTIFY_CLIENT_ID_KEY, "").trim();
}
export function isValidSpotifyClientId(value: string): boolean {
return /^[0-9a-f]{32}$/i.test(value.trim());
}
/** The stored sign-in, or null when signed out or signed in with another client ID. */
export function readSpotifyAccount(): SpotifyAccount | null {
let raw: any;
try {
raw = JSON.parse(getStringSetting(SPOTIFY_ACCOUNT_KEY, "") || "null");
} catch {
return null;
}
if (!raw || typeof raw.refreshToken !== "string" || !raw.refreshToken) return null;
const clientId = spotifyClientId();
if (!clientId || raw.clientId !== clientId) return null;
return {
clientId,
accessToken: String(raw.accessToken ?? ""),
refreshToken: raw.refreshToken,
expiresAtMs: Number(raw.expiresAtMs) || 0,
userId: String(raw.userId ?? ""),
displayName: String(raw.displayName ?? ""),
product: String(raw.product ?? ""),
};
}
export function saveSpotifyAccount(account: SpotifyAccount | null): void {
setStringSetting(SPOTIFY_ACCOUNT_KEY, account ? JSON.stringify(account) : "");
}
export function spotifyAuthorizeUrl(clientId: string, codeChallenge: string, state: string): string {
return `${SPOTIFY_ACCOUNTS_BASE}/authorize${buildQuery({
response_type: "code",
client_id: clientId,
scope: SPOTIFY_SCOPES.join(" "),
redirect_uri: SPOTIFY_REDIRECT_URI,
state,
code_challenge_method: "S256",
code_challenge: codeChallenge,
})}`;
}
/** The authorization code from Spotify's redirect; throws with a message for the glasses otherwise. */
export function parseSpotifyRedirect(url: string, expectedState: string): string {
const params = parseQueryString(url);
if (params.error) {
throw new Error(params.error === "access_denied" ? "Spotify sign-in was declined." : `Spotify sign-in failed: ${params.error}`);
}
if (params.state !== expectedState) throw new Error("Spotify sign-in failed: the reply didn't match this sign-in. Try again.");
if (!params.code) throw new Error("Spotify sign-in failed: no authorization code in the reply.");
return params.code;
}
/** Query parameters of a URL (not URLSearchParams, which the NativeScript runtime mishandles). */
export function parseQueryString(url: string): Record<string, string> {
const query = url.split("#")[0]!.split("?").slice(1).join("?");
const params: Record<string, string> = {};
for (const part of query.split("&")) {
if (!part) continue;
const separator = part.indexOf("=");
const key = separator < 0 ? part : part.slice(0, separator);
const value = separator < 0 ? "" : part.slice(separator + 1);
try {
params[decodeURIComponent(key.replace(/\+/g, " "))] = decodeURIComponent(value.replace(/\+/g, " "));
} catch {
// Malformed escapes: skip the parameter.
}
}
return params;
}
/** POST to Spotify's token endpoint (authorization_code or refresh_token grant). */
export async function requestSpotifyTokens(
fetchImpl: SpotifyFetch,
params: Record<string, string>,
nowMs: number,
): Promise<SpotifyTokens> {
let status = 0;
let body = "";
try {
const response = await fetchImpl(`${SPOTIFY_ACCOUNTS_BASE}/api/token`, {
method: "POST",
headers: { "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json" },
body: buildQuery(params).slice(1),
});
status = response.status;
body = await response.text();
} catch (error) {
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
}
let parsed: any = null;
try {
parsed = JSON.parse(body);
} catch {
// Reported below.
}
if (status >= 200 && status < 300 && typeof parsed?.access_token === "string") {
const expiresIn = Number(parsed.expires_in);
return {
accessToken: parsed.access_token,
// Spotify may rotate the refresh token; "" keeps the current one.
refreshToken: typeof parsed.refresh_token === "string" ? parsed.refresh_token : "",
expiresAtMs: nowMs + (Number.isFinite(expiresIn) && expiresIn > 0 ? expiresIn : 3600) * 1000,
};
}
const error = typeof parsed?.error === "string" ? parsed.error : "";
const description = typeof parsed?.error_description === "string" ? parsed.error_description : "";
if (error === "invalid_grant" && params.grant_type === "refresh_token") {
throw new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, error);
}
if (error === "invalid_client") {
throw new SpotifyApiError("Spotify doesn't recognize the client ID. Check it under Spotify in the Music app menu.", status, error);
}
throw new SpotifyApiError(`Spotify sign-in failed: ${description || error || `HTTP ${status}`}`, status, error);
}
/**
* Access tokens for the stored account: the current one until it nears
* expiry, then one refresh shared by every caller waiting on it. A refresh
* Spotify rejects (revoked access) signs out.
*/
export class SpotifySession implements SpotifyTokenSource {
private refreshing: Promise<string> | null = null;
constructor(
private readonly fetchImpl: SpotifyFetch,
private readonly now: () => number = Date.now,
) {}
accessToken(forceRefresh: boolean): Promise<string> {
const account = readSpotifyAccount();
if (!account) return Promise.reject(new SpotifyApiError("Not signed in to Spotify.", 401, "signed_out"));
if (!forceRefresh && account.accessToken && account.expiresAtMs - EXPIRY_MARGIN_MS > this.now()) {
return Promise.resolve(account.accessToken);
}
this.refreshing ??= this.refresh(account).finally(() => {
this.refreshing = null;
});
return this.refreshing;
}
private async refresh(account: SpotifyAccount): Promise<string> {
let tokens: SpotifyTokens;
try {
tokens = await requestSpotifyTokens(this.fetchImpl, {
grant_type: "refresh_token",
refresh_token: account.refreshToken,
client_id: account.clientId,
}, this.now());
} catch (error) {
if (error instanceof SpotifyApiError && error.reason === "invalid_grant" && readSpotifyAccount()?.refreshToken === account.refreshToken) {
saveSpotifyAccount(null);
}
throw error;
}
// A sign-out or another sign-in while this was in flight wins.
if (readSpotifyAccount()?.refreshToken !== account.refreshToken) {
throw new SpotifyApiError("Spotify sign-in changed; try again.", 401, "signed_out");
}
saveSpotifyAccount({
...account,
accessToken: tokens.accessToken,
refreshToken: tokens.refreshToken || account.refreshToken,
expiresAtMs: tokens.expiresAtMs,
});
return tokens.accessToken;
}
}
/**
* The whole sign-in: the browser sheet, the code exchange and the profile
* lookup; stores and returns the account. Rejects with "cancelled" when the
* user closes the sheet.
*/
export async function signInToSpotify(
fetchImpl: SpotifyFetch,
webSignIn: WebSignIn,
now: () => number = Date.now,
): Promise<SpotifyAccount> {
const clientId = spotifyClientId();
if (!isValidSpotifyClientId(clientId)) throw new Error("Enter your Spotify client ID first.");
const { verifier, challenge } = webSignIn.createPkce();
const state = webSignIn.randomState();
const redirect = await webSignIn.open(spotifyAuthorizeUrl(clientId, challenge, state), SPOTIFY_REDIRECT_PORT, SPOTIFY_REDIRECT_PATH);
const code = parseSpotifyRedirect(redirect, state);
const tokens = await requestSpotifyTokens(fetchImpl, {
grant_type: "authorization_code",
code,
redirect_uri: SPOTIFY_REDIRECT_URI,
client_id: clientId,
code_verifier: verifier,
}, now());
const user = await new SpotifyClient(fetchImpl, { accessToken: async () => tokens.accessToken }).me();
const account: SpotifyAccount = {
clientId,
accessToken: tokens.accessToken,
refreshToken: tokens.refreshToken,
expiresAtMs: tokens.expiresAtMs,
userId: user.id,
displayName: user.displayName,
product: user.product,
};
if (!account.refreshToken) throw new Error("Spotify sign-in failed: no refresh token in the reply.");
saveSpotifyAccount(account);
return account;
}
+162
View File
@@ -0,0 +1,162 @@
import type { MediaBrowseItem, MediaBrowserApp, MediaBrowserBridge } from "./media-types";
import { SPOTIFY_APP_NAME, type SpotifyPlayer } from "./spotify-player";
import type { SpotifyClient, SpotifyItem, SpotifyPlayRequest } from "./spotify-web-api";
const ROOT_ROWS: MediaBrowseItem[] = [
{ mediaId: "recent", title: "Recently played", subtitle: "", browsable: true, playable: false },
{ mediaId: "playlists", title: "Playlists", subtitle: "", browsable: true, playable: false },
{ mediaId: "liked", title: "Liked Songs", subtitle: "", browsable: true, playable: true },
{ mediaId: "albums", title: "Albums", subtitle: "", browsable: true, playable: false },
{ mediaId: "artists", title: "Artists", subtitle: "", browsable: true, playable: false },
];
/** Separates a track's URI from the context it plays in ("" for none, "liked" for Liked Songs). */
const TRACK_CONTEXT_SEPARATOR = "|";
export type SpotifyLibraryDeps = {
client: SpotifyClient;
/** Spotify's app id on this platform, which names the library in the browse picker. */
packageName: string;
player: SpotifyPlayer;
isSignedIn(): boolean;
/** The signed-in user's id (Liked Songs' context URI names it). */
userId(): string;
};
/**
* The signed-in user's Spotify library as a browse tree (Web API):
*
* root -> recent | playlists | liked | albums | artists
* playlist:<id>, album:<id> -> tracks (browsable and playable)
* artist:<id> -> albums
* track:<uri>|<context> -> plays the track within its context
*
* Only playlists the user owns or collaborates on can list their tracks
* (development-mode client IDs, since February 2026); others are play-only.
*/
export class SpotifyLibrary implements MediaBrowserBridge {
private connected = false;
/** Bumped on every connect/disconnect; answers to older requests are dropped. */
private generation = 0;
/** This library's row in the browse picker. */
readonly app: MediaBrowserApp;
constructor(private readonly deps: SpotifyLibraryDeps) {
this.app = { packageName: deps.packageName, serviceClass: "", appName: SPOTIFY_APP_NAME };
}
listBrowsableApps(): MediaBrowserApp[] {
return this.deps.isSignedIn() ? [this.app] : [];
}
connect(app: MediaBrowserApp): Promise<string> {
this.disconnect();
if (app.packageName !== this.app.packageName) return Promise.reject(new Error("Not a Spotify library"));
if (!this.deps.isSignedIn()) return Promise.reject(new Error("Sign in under Spotify in the Music app menu first."));
this.connected = true;
return Promise.resolve("root");
}
async browse(parentId: string): Promise<MediaBrowseItem[]> {
if (!this.connected) throw new Error("Not connected");
const generation = this.generation;
const rows = await this.list(parentId);
if (generation !== this.generation) throw new Error("Disconnected");
return rows;
}
playFromMediaId(mediaId: string): void {
if (!this.connected) return;
const request = this.playRequest(mediaId);
if (request) void this.deps.player.startPlayback(request);
}
disconnect(): void {
this.connected = false;
this.generation++;
}
/** What playing a browse row starts; null for rows that don't play. */
playRequest(mediaId: string): SpotifyPlayRequest | null {
if (mediaId === "liked") return { contextUri: this.likedContextUri() };
const [kind, ...rest] = mediaId.split(":");
const id = rest.join(":");
switch (kind) {
case "playlist":
case "album":
case "artist":
return id ? { contextUri: `spotify:${kind}:${id}` } : null;
case "track": {
const separator = id.lastIndexOf(TRACK_CONTEXT_SEPARATOR);
const uri = separator < 0 ? id : id.slice(0, separator);
const context = separator < 0 ? "" : id.slice(separator + 1);
if (!uri) return null;
if (context === "liked") return { contextUri: this.likedContextUri(), offsetUri: uri };
return context ? { contextUri: context, offsetUri: uri } : { uris: [uri] };
}
default:
return null;
}
}
private async list(parentId: string): Promise<MediaBrowseItem[]> {
const client = this.deps.client;
if (parentId === "root") return ROOT_ROWS;
if (parentId === "recent") {
// The history repeats tracks; keep each one's latest play.
const seen = new Set<string>();
return (await client.recentlyPlayed())
.filter((played) => !seen.has(played.item.uri) && seen.add(played.item.uri))
.map((played) => trackRow(played.item, played.contextUri));
}
if (parentId === "playlists") {
const userId = this.deps.userId();
return (await client.playlists()).map((playlist) => {
const listable = playlist.collaborative || playlist.ownerId === userId;
return {
mediaId: `playlist:${playlist.id}`,
title: playlist.name,
subtitle: listable ? "" : playlist.ownerName,
browsable: listable,
playable: true,
};
});
}
if (parentId === "liked") return (await client.savedTracks()).map((item) => trackRow(item, "liked"));
if (parentId === "albums") return (await client.savedAlbums()).map(albumRow);
if (parentId === "artists") {
return (await client.followedArtists()).map((artist) => ({
mediaId: `artist:${artist.id}`,
title: artist.name,
subtitle: "",
browsable: true,
playable: true,
}));
}
const [kind, ...rest] = parentId.split(":");
const id = rest.join(":");
if (kind === "playlist") return (await client.playlistItems(id)).map((item) => trackRow(item, `spotify:playlist:${id}`));
if (kind === "album") return (await client.albumTracks(id)).map((item) => trackRow(item, `spotify:album:${id}`));
if (kind === "artist") return (await client.artistAlbums(id)).map(albumRow);
throw new Error(`Unknown Spotify library item: ${parentId}`);
}
private likedContextUri(): string {
return `spotify:user:${this.deps.userId()}:collection`;
}
}
function trackRow(item: SpotifyItem, contextUri: string): MediaBrowseItem {
return {
mediaId: `track:${item.uri}${contextUri ? `${TRACK_CONTEXT_SEPARATOR}${contextUri}` : ""}`,
title: item.name,
subtitle: item.artists,
browsable: false,
playable: true,
};
}
function albumRow(album: { id: string; name: string; artists: string }): MediaBrowseItem {
return { mediaId: `album:${album.id}`, title: album.name, subtitle: album.artists, browsable: true, playable: true };
}
+574
View File
@@ -0,0 +1,574 @@
import type { GrayImage } from "../graphics/image";
import {
DEFAULT_MEDIA_STATE,
interpolateMediaState,
type MediaControllerBridge,
type MediaControllerState,
type MediaCustomAction,
type MediaQueueItem,
} from "./media-types";
import {
SpotifyApiError,
type SpotifyClient,
type SpotifyDevice,
type SpotifyItem,
type SpotifyPlayback,
type SpotifyPlayRequest,
type SpotifyRepeat,
} from "./spotify-web-api";
export const SPOTIFY_APP_NAME = "Spotify";
/** Spotify's iOS bundle id: the player's packageName in the media state on iOS. */
export const SPOTIFY_IOS_BUNDLE_ID = "com.spotify.client";
/** Spotify's Android package: its media session's owner, which the Web API player reports too. */
export const SPOTIFY_ANDROID_PACKAGE = "com.spotify.music";
/**
* Poll intervals while something listens. The Web API has no push channel, so
* a track change shows up within one interval (sooner at a track's end, which
* is polled for directly); requests count against the client ID's quota.
*/
const POLL_PLAYING_MS = 5_000;
const POLL_PAUSED_MS = 10_000;
const POLL_IDLE_MS = 20_000;
const POLL_ERROR_MS = 30_000;
/** Re-read soon after a command: Spotify takes a moment to report its effect. */
const SETTLE_POLL_MS = 600;
const SETTLE_POLLS = 2;
const NOTICE_MS = 8_000;
/** Re-read the queue this often while the Music app shows it, besides on every track change. */
const QUEUE_MAX_AGE_MS = 30_000;
/** At most this often, even on track changes (and after a failed read). */
const QUEUE_MIN_INTERVAL_MS = 5_000;
/** Album art: the smallest of Spotify's sizes (640/300/64) at least this wide. */
const ART_MIN_WIDTH = 128;
const REPEAT_NEXT: Record<SpotifyRepeat, SpotifyRepeat> = { off: "context", context: "track", track: "off" };
const REPEAT_LABEL: Record<SpotifyRepeat, string> = { off: "Repeat: off", context: "Repeat: all", track: "Repeat: one" };
export type SpotifyPlayerDeps = {
client: SpotifyClient;
/** The state's packageName: Spotify's app id on this platform. */
packageName: string;
isSignedIn(): boolean;
/** Fetch an album-art image's encoded bytes; null on failure. */
download(url: string): Promise<Uint8Array | null>;
/** Encoded image bytes to the shared photo tone, fitted within maxSize; null when undecodable. */
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
/** The phone's output volume, for playback on this phone (Spotify won't set a phone's volume). */
systemVolume: { get(): number; set(percent: number): void };
now?: () => number;
};
/**
* Spotify Connect playback (on this phone or any of the user's devices)
* through the Web API: state from polling GET /me/player while something
* listens, commands as API calls whose failures (no device, no Premium)
* become a short notice in the state.
*/
export class SpotifyPlayer implements MediaControllerBridge {
readonly accessPrompt = "Sign in to Spotify from the Music app menu to show and control Spotify.";
readonly accessLabel = "Spotify sign-in needed";
readonly accessAction = "open the menu";
/** Jumping skips forward through the queue, so the chosen track ends up first. */
readonly skipRebuildsQueue = true;
/** When Spotify was last seen playing (epoch ms); 0 = not this run. */
lastPlayingAtMs = 0;
private readonly now: () => number;
private readonly listeners = new Set<(state: MediaControllerState) => void>();
private playback: SpotifyPlayback | null = null;
private fetchedAtMs = 0;
private pollTimer: ReturnType<typeof setTimeout> | null = null;
private polling: Promise<void> | null = null;
private pollAgain = false;
private settlePolls = 0;
private lastSignature = "";
private notice = "";
private noticeUntilMs = 0;
private noticeTimer: ReturnType<typeof setTimeout> | null = null;
private upcoming: SpotifyItem[] = [];
/** The item the queue was read with; "" = never read. */
private queueCurrentUri = "";
/** When the queue was last read, or a read failed. */
private queueFetchedAtMs = 0;
private queueLoading = false;
private art: { url: string; bytes: Uint8Array | null; decoded: Map<number, GrayImage | null> } | null = null;
private artVersion = 0;
private liked: { uri: string; value: boolean | null } | null = null;
private volumeTarget: number | null = null;
private volumeSending = false;
constructor(private readonly deps: SpotifyPlayerDeps) {
this.now = deps.now ?? Date.now;
}
onStateChange(listener: (state: MediaControllerState) => void): () => void {
this.listeners.add(listener);
listener(this.snapshot());
if (this.listeners.size === 1) this.schedulePoll(0);
return () => {
this.listeners.delete(listener);
if (!this.listeners.size && !this.settlePolls) this.clearPollTimer();
};
}
snapshot(): MediaControllerState {
const notice = this.currentNotice();
if (!this.deps.isSignedIn()) {
return { ...DEFAULT_MEDIA_STATE, status: "Not signed in to Spotify.", notice };
}
const playback = this.playback;
if (!playback) {
return {
...DEFAULT_MEDIA_STATE,
accessEnabled: true,
packageName: this.deps.packageName,
appName: SPOTIFY_APP_NAME,
status: "Nothing is playing on Spotify.",
notice,
};
}
const item = playback.item;
const restricted = playback.device?.restricted === true;
const allowed = (action: string) => !restricted && !playback.disallows.includes(action);
const state: MediaControllerState = {
available: true,
accessEnabled: true,
packageName: this.deps.packageName,
appName: spotifyAppName(playback.device),
playbackState: playback.playing ? "playing" : "paused",
title: item?.name ?? (playback.playingType === "ad" ? "Advertisement" : ""),
artist: item?.artists ?? "",
album: item?.album ?? "",
positionMs: item ? playback.progressMs : -1,
durationMs: item?.durationMs ?? -1,
playbackSpeed: playback.playing ? 1 : 0,
status: "",
canPlayPause: allowed(playback.playing ? "pausing" : "resuming"),
canSkipNext: allowed("skipping_next"),
canSkipPrevious: allowed("skipping_prev"),
artVersion: this.artVersion,
customActions: restricted ? [] : this.customActions(playback),
notice,
};
return interpolateMediaState(state, this.fetchedAtMs, this.now());
}
async start(): Promise<void> {}
async stop(): Promise<void> {}
async playPause(): Promise<void> {
const playback = this.playback;
if (!playback?.playing) {
await this.startPlayback({});
return;
}
await this.command(() => this.deps.client.pause(), () => {
playback.progressMs = this.snapshot().positionMs;
playback.playing = false;
this.fetchedAtMs = this.now();
});
}
async skipNext(): Promise<void> {
await this.command(() => this.deps.client.next());
}
async skipPrevious(): Promise<void> {
await this.command(() => this.deps.client.previous());
}
/** Signed in; whether anything is playing is in the state. */
hasAccess(): boolean {
return this.deps.isSignedIn();
}
/** Sign-in happens from the Music app menu, which shows its progress. */
async requestAccess(): Promise<boolean> {
return this.hasAccess();
}
/**
* Queue ids are positions in getQueue (0 = the current item). The Web API
* has no jump, so this skips forward that many times.
*/
async skipToQueueItem(id: number): Promise<void> {
const steps = Math.round(id);
if (steps <= 0 || steps > this.upcoming.length) return;
await this.command(async () => {
for (let step = 0; step < steps; step++) await this.deps.client.next();
}, () => {
const target = this.upcoming[steps - 1];
if (this.playback && target) {
this.playback.item = target;
this.playback.progressMs = 0;
this.fetchedAtMs = this.now();
this.upcoming = this.upcoming.slice(steps);
this.queueCurrentUri = target.uri;
this.itemChanged(target);
}
});
}
getMediaVolumePercent(): number {
const device = this.playback?.device ?? null;
if (!usesDeviceVolume(device)) return this.deps.systemVolume.get();
return this.volumeTarget ?? device!.volumePercent;
}
setMediaVolumePercent(volumePercent: number): void {
const device = this.playback?.device ?? null;
const volume = Math.max(0, Math.min(100, Math.round(volumePercent)));
if (!usesDeviceVolume(device)) {
this.deps.systemVolume.set(volume);
return;
}
// Scrolling sends a value per step: keep one request in flight and send the latest after it.
device!.volumePercent = volume;
this.volumeTarget = volume;
if (!this.volumeSending) void this.sendVolume(device!.id);
}
getAlbumArt(maxSize: number): GrayImage | null {
const art = this.art;
if (!art?.bytes) return null;
const size = Math.round(maxSize);
if (!art.decoded.has(size)) art.decoded.set(size, this.deps.decodeArt(art.bytes, size));
return art.decoded.get(size) ?? null;
}
/**
* The current item, then what Spotify will play next. Read on demand (the
* Music app asks while it paints), so nothing else pays for the request; a
* stale list stays up, minus the items already played, until the new one
* arrives.
*/
getQueue(): MediaQueueItem[] {
const current = this.playback?.item;
if (!current) return [];
const age = this.now() - this.queueFetchedAtMs;
if (!this.queueLoading && (age > QUEUE_MAX_AGE_MS || (this.queueCurrentUri !== current.uri && age > QUEUE_MIN_INTERVAL_MS))) {
void this.loadQueue();
}
if (!this.queueCurrentUri) return [];
if (this.queueCurrentUri !== current.uri) {
const index = this.upcoming.findIndex((item) => item.uri === current.uri);
if (index >= 0) {
this.upcoming = this.upcoming.slice(index + 1);
this.queueCurrentUri = current.uri;
}
}
return [current, ...this.upcoming].map((item, index) => ({
id: index,
title: item.name,
subtitle: item.artists,
active: index === 0,
}));
}
async sendCustomAction(action: string): Promise<void> {
const playback = this.playback;
if (!playback) return;
const client = this.deps.client;
if (action === "shuffle") {
const shuffle = !playback.shuffle;
await this.command(() => client.setShuffle(shuffle), () => {
playback.shuffle = shuffle;
// Shuffling reorders what comes next.
this.queueFetchedAtMs = 0;
});
} else if (action === "repeat") {
const repeat = REPEAT_NEXT[playback.repeat];
await this.command(() => client.setRepeat(repeat), () => {
playback.repeat = repeat;
});
} else if (action === "like") {
const liked = this.liked;
if (!liked || liked.value === null) return;
const value = !liked.value;
await this.command(() => (value ? client.saveToLibrary([liked.uri]) : client.removeFromLibrary([liked.uri])), () => {
if (this.liked === liked) liked.value = value;
});
}
}
async refresh(): Promise<void> {
await this.poll();
}
/**
* Play something (a browsed item, a search result) or, with an empty
* request, resume. With no active device, starts on one the account has
* open (preferring a phone). Resolves to whether it worked; a failure is
* left as the state's notice.
*/
async startPlayback(request: SpotifyPlayRequest): Promise<boolean> {
const client = this.deps.client;
const playback = this.playback;
const ok = await this.command(async () => {
try {
await client.play(request);
} catch (error) {
if (!(error instanceof SpotifyApiError) || error.reason !== "NO_ACTIVE_DEVICE") throw error;
const device = pickDevice(await client.devices());
if (!device) throw error;
await client.play(request, device.id);
}
}, () => {
this.lastPlayingAtMs = this.now();
if (playback && this.playback === playback && !request.contextUri && !request.uris) {
playback.playing = true;
this.fetchedAtMs = this.now();
}
});
return ok;
}
/** The account changed (signed in, out, or as someone else): forget the old account's state. */
accountChanged(): void {
this.playback = null;
this.upcoming = [];
this.queueCurrentUri = "";
this.art = null;
this.artVersion++;
this.liked = null;
this.lastPlayingAtMs = 0;
this.emit(true);
if (this.listeners.size) this.schedulePoll(0);
}
/** Show a message about a failed command in the state's notice for a few seconds. */
showNotice(message: string): void {
this.notice = message;
this.noticeUntilMs = this.now() + NOTICE_MS;
if (this.noticeTimer !== null) clearTimeout(this.noticeTimer);
this.noticeTimer = setTimeout(() => {
this.noticeTimer = null;
this.emit(true);
}, NOTICE_MS + 50);
this.emit(true);
}
private currentNotice(): string {
return this.notice && this.now() < this.noticeUntilMs ? this.notice : "";
}
private customActions(playback: SpotifyPlayback): MediaCustomAction[] {
if (!playback.item) return [];
const actions: MediaCustomAction[] = [];
if (!playback.disallows.includes("toggling_shuffle")) {
actions.push({ action: "shuffle", name: playback.shuffle ? "Shuffle: on" : "Shuffle: off" });
}
if (!playback.disallows.includes("toggling_repeat_context") || !playback.disallows.includes("toggling_repeat_track")) {
actions.push({ action: "repeat", name: REPEAT_LABEL[playback.repeat] });
}
const liked = this.liked;
if (playback.item.type === "track" && liked?.uri === playback.item.uri && liked.value !== null) {
actions.push({ action: "like", name: liked.value ? "Unlike song" : "Like song" });
}
return actions;
}
/** Run a command; on success apply its expected effect at once, then re-read to confirm. */
private async command(run: () => Promise<void>, applied?: () => void): Promise<boolean> {
let ok = false;
try {
await run();
ok = true;
applied?.();
this.emit(true);
} catch (error) {
this.showNotice(errorMessage(error));
}
this.settlePolls = SETTLE_POLLS;
this.schedulePoll(SETTLE_POLL_MS);
return ok;
}
private async sendVolume(deviceId: string): Promise<void> {
this.volumeSending = true;
try {
while (this.volumeTarget !== null) {
const volume = this.volumeTarget;
try {
await this.deps.client.setVolume(volume, deviceId);
} catch (error) {
this.showNotice(errorMessage(error));
}
if (this.volumeTarget === volume) this.volumeTarget = null;
}
} finally {
this.volumeSending = false;
}
}
private poll(): Promise<void> {
if (this.polling) {
this.pollAgain = true;
return this.polling;
}
this.polling = this.pollOnce().finally(() => {
this.polling = null;
if (this.pollAgain) {
this.pollAgain = false;
void this.poll();
}
});
return this.polling;
}
private async pollOnce(): Promise<void> {
this.clearPollTimer();
if (!this.deps.isSignedIn()) {
if (this.playback) {
this.playback = null;
this.emit(true);
}
return;
}
let delay: number;
try {
const playback = await this.deps.client.playback();
this.fetchedAtMs = this.now();
this.applyPlayback(playback);
delay = this.nextPollDelay(playback);
} catch (error) {
if (!(error instanceof SpotifyApiError) || error.reason !== "signed_out") {
console.warn(`[spotify] playback read failed: ${error}`);
}
// Revoked access signs out (SpotifySession); the account listener resets the state.
delay = POLL_ERROR_MS;
}
if (this.settlePolls > 0) {
this.settlePolls--;
delay = Math.min(delay, 1_500);
}
this.emit(false);
this.schedulePoll(delay);
}
private applyPlayback(playback: SpotifyPlayback | null): void {
const previousUri = this.playback?.item?.uri ?? "";
this.playback = playback;
if (playback?.playing) this.lastPlayingAtMs = this.now();
const item = playback?.item ?? null;
if ((item?.uri ?? "") !== previousUri) this.itemChanged(item);
}
/** A new item: fetch its art and whether it is liked. */
private itemChanged(item: SpotifyItem | null): void {
const url = item ? pickArtUrl(item) : "";
if (url !== (this.art?.url ?? "")) {
this.artVersion++;
this.art = url ? { url, bytes: null, decoded: new Map() } : null;
if (url) void this.loadArt(url);
}
this.liked = item?.type === "track" ? { uri: item.uri, value: null } : null;
if (item?.type === "track") void this.loadLiked(item.uri);
}
private async loadArt(url: string): Promise<void> {
let bytes: Uint8Array | null = null;
try {
bytes = await this.deps.download(url);
} catch (error) {
console.warn(`[spotify] album art download failed: ${error}`);
}
if (!bytes?.length || this.art?.url !== url) return;
this.art.bytes = bytes;
this.artVersion++;
this.emit(true);
}
private async loadLiked(uri: string): Promise<void> {
try {
const [value] = await this.deps.client.libraryContains([uri]);
if (this.liked?.uri !== uri) return;
this.liked.value = Boolean(value);
this.emit(true);
} catch (error) {
console.warn(`[spotify] library check failed: ${error}`);
}
}
private async loadQueue(): Promise<void> {
this.queueLoading = true;
try {
const queue = await this.deps.client.queue();
this.upcoming = queue.upcoming;
this.queueCurrentUri = queue.current?.uri ?? "";
this.queueFetchedAtMs = this.now();
this.emit(true);
} catch (error) {
console.warn(`[spotify] queue read failed: ${error}`);
this.queueFetchedAtMs = this.now();
} finally {
this.queueLoading = false;
}
}
private nextPollDelay(playback: SpotifyPlayback | null): number {
if (!playback) return POLL_IDLE_MS;
if (!playback.playing) return POLL_PAUSED_MS;
const durationMs = playback.item?.durationMs ?? -1;
// Catch the change to the next track as it happens.
const remaining = durationMs > 0 ? durationMs - playback.progressMs + 700 : POLL_PLAYING_MS;
return Math.max(1_000, Math.min(POLL_PLAYING_MS, remaining));
}
private schedulePoll(delayMs: number): void {
if (!this.listeners.size && !this.settlePolls) return;
this.clearPollTimer();
this.pollTimer = setTimeout(() => {
this.pollTimer = null;
void this.poll();
}, delayMs);
}
private clearPollTimer(): void {
if (this.pollTimer !== null) clearTimeout(this.pollTimer);
this.pollTimer = null;
}
/** Tell listeners; unforced, only when something they draw (other than the advancing position) changed. */
private emit(force: boolean): void {
if (!this.listeners.size) return;
const state = this.snapshot();
const signature = JSON.stringify({ ...state, positionMs: 0 });
if (!force && signature === this.lastSignature) return;
this.lastSignature = signature;
for (const listener of this.listeners) listener(state);
}
}
/** "Spotify", or "Spotify · <device>" when it plays elsewhere than a phone. */
export function spotifyAppName(device: SpotifyDevice | null): string {
if (!device?.name || device.type === "Smartphone") return SPOTIFY_APP_NAME;
return `${SPOTIFY_APP_NAME} · ${device.name}`;
}
/** Spotify sets a speaker's or computer's volume; a phone's (this one) goes through the system volume. */
function usesDeviceVolume(device: SpotifyDevice | null): boolean {
return Boolean(device && device.type !== "Smartphone" && device.supportsVolume && device.volumePercent >= 0 && device.id);
}
/** A device to start playback on when none is active: an open one, a phone first. */
export function pickDevice(devices: SpotifyDevice[]): SpotifyDevice | null {
const usable = devices.filter((device) => device.id && !device.restricted);
return usable.find((device) => device.active)
?? usable.find((device) => device.type === "Smartphone")
?? usable[0]
?? null;
}
function pickArtUrl(item: SpotifyItem): string {
const images = [...item.images].sort((a, b) => a.width - b.width);
return (images.find((image) => image.width >= ART_MIN_WIDTH) ?? images[images.length - 1])?.url ?? "";
}
export function errorMessage(error: unknown): string {
if (error instanceof Error) return error.message;
return String(error);
}
+163
View File
@@ -0,0 +1,163 @@
import type { GrayImage } from "../graphics/image";
import { onSettingsStoreChanged } from "./settings-store";
import {
readSpotifyAccount,
saveSpotifyAccount,
signInToSpotify,
SPOTIFY_ACCOUNT_KEY,
SPOTIFY_CLIENT_ID_KEY,
SpotifySession,
type SpotifyAccount,
type WebSignIn,
} from "./spotify-auth";
import { SpotifyLibrary } from "./spotify-library";
import { errorMessage, SpotifyPlayer } from "./spotify-player";
import { SpotifyClient, type SpotifyFetch, type SpotifySearchType } from "./spotify-web-api";
/** Give up on a sign-in nobody finished on the phone. */
const SIGN_IN_TIMEOUT_MS = 10 * 60_000;
/** What the platform supplies (spotify.ios.ts, spotify.ts). */
export type SpotifyPlatform = {
/** Spotify's app id on this platform (SPOTIFY_IOS_BUNDLE_ID / SPOTIFY_ANDROID_PACKAGE). */
packageName: string;
fetch: SpotifyFetch;
webSignIn: WebSignIn;
download(url: string): Promise<Uint8Array | null>;
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
systemVolume: { get(): number; set(percent: number): void };
};
export type SpotifySignInState =
| { phase: "idle"; error: string }
/** The sign-in sheet is open on the phone, or waiting for Faceclaw to come to the foreground. */
| { phase: "waiting"; error: "" };
/**
* Spotify for the Music app, the media bridges and the assistant: the
* account and its sign-in, the Web API client, the player and the library.
*/
export class SpotifyService {
readonly client: SpotifyClient;
readonly player: SpotifyPlayer;
readonly library: SpotifyLibrary;
private signInState: SpotifySignInState = { phase: "idle", error: "" };
private readonly listeners = new Set<() => void>();
private accountKey: string;
constructor(private readonly platform: SpotifyPlatform) {
const isSignedIn = () => readSpotifyAccount() !== null;
this.client = new SpotifyClient(platform.fetch, new SpotifySession(platform.fetch));
this.player = new SpotifyPlayer({
client: this.client,
packageName: platform.packageName,
isSignedIn,
download: (url) => platform.download(url),
decodeArt: (bytes, maxSize) => platform.decodeArt(bytes, maxSize),
systemVolume: platform.systemVolume,
});
this.library = new SpotifyLibrary({
client: this.client,
packageName: platform.packageName,
player: this.player,
isSignedIn,
userId: () => readSpotifyAccount()?.userId ?? "",
});
this.accountKey = accountIdentity(readSpotifyAccount());
onSettingsStoreChanged((key) => {
if (key !== SPOTIFY_ACCOUNT_KEY && key !== SPOTIFY_CLIENT_ID_KEY) return;
// Token refreshes rewrite the account too; only a different sign-in resets the player.
const identity = accountIdentity(readSpotifyAccount());
if (identity !== this.accountKey) {
this.accountKey = identity;
this.player.accountChanged();
}
this.emit();
});
}
account(): SpotifyAccount | null {
return readSpotifyAccount();
}
signInStatus(): SpotifySignInState {
return this.signInState;
}
/** Repaint hook for the sign-in menu: account or sign-in progress changed. */
onChange(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
/**
* Open Spotify's sign-in page on the phone (it waits for Faceclaw to be in
* the foreground); the outcome lands in signInStatus().
*/
async signIn(): Promise<void> {
if (this.signInState.phase === "waiting") return;
this.signInState = { phase: "waiting", error: "" };
this.emit();
const timer = setTimeout(() => this.platform.webSignIn.cancel(), SIGN_IN_TIMEOUT_MS);
try {
await signInToSpotify(this.platform.fetch, this.platform.webSignIn);
this.signInState = { phase: "idle", error: "" };
} catch (error) {
const message = errorMessage(error);
this.signInState = { phase: "idle", error: message === "cancelled" ? "" : message };
} finally {
clearTimeout(timer);
this.emit();
}
}
cancelSignIn(): void {
this.platform.webSignIn.cancel();
}
signOut(): void {
saveSpotifyAccount(null);
this.signInState = { phase: "idle", error: "" };
this.emit();
}
/**
* Search Spotify and play the best match: a track, or the album, artist or
* playlist when the request names that kind. Resolves to what started, or
* rejects with why not.
*/
async playSearch(query: string, kind: SpotifySearchType = "track"): Promise<string> {
if (!readSpotifyAccount()) throw new Error("Not signed in to Spotify. Sign in under Spotify in the Music app menu on the glasses.");
const results = await this.client.search(query, [kind], 3);
let request: { contextUri?: string; uris?: string[] } | null = null;
let description = "";
if (kind === "track" && results.tracks[0]) {
const track = results.tracks[0];
request = { uris: [track.uri] };
description = `${track.name}${track.artists ? ` by ${track.artists}` : ""}`;
} else if (kind === "album" && results.albums[0]) {
const album = results.albums[0];
request = { contextUri: album.uri };
description = `the album ${album.name}${album.artists ? ` by ${album.artists}` : ""}`;
} else if (kind === "artist" && results.artists[0]) {
request = { contextUri: results.artists[0].uri };
description = results.artists[0].name;
} else if (kind === "playlist" && results.playlists[0]) {
request = { contextUri: results.playlists[0].uri };
description = `the playlist ${results.playlists[0].name}`;
}
if (!request) throw new Error(`Spotify found no ${kind} matching "${query}".`);
if (!(await this.player.startPlayback(request))) {
throw new Error(this.player.snapshot().notice || "Spotify didn't start playback.");
}
return description;
}
private emit(): void {
for (const listener of this.listeners) listener();
}
}
function accountIdentity(account: SpotifyAccount | null): string {
return account ? `${account.clientId}/${account.userId}` : "";
}
+474
View File
@@ -0,0 +1,474 @@
/**
* Spotify Web API over HTTPS with the signed-in user's token: the player
* (what is playing on any of the user's Spotify Connect devices, and control
* of it), the user's library, and search, normalized into the small shapes
* the Music app uses. No NativeScript imports, so the Node tests drive it with
* a fake fetch.
*
* Every personal client ID is a development-mode app, which since February
* 2026 means: the client ID's owner needs Premium, playback control needs
* Premium, at most five allowlisted users, only owned or collaborative
* playlists list their items, no batch lookups, and search and artist albums
* return at most 10 results per page.
*/
export const SPOTIFY_API_BASE = "https://api.spotify.com/v1";
const HTTP_TIMEOUT_MS = 12_000;
/** The most a library listing gathers (a page is 50); enough to scroll through on the glasses. */
const MAX_LIST_ITEMS = 500;
/** The subset of fetch this client needs (fetchTextWithUserAgent in the app). */
export type SpotifyFetch = (
url: string,
init: { method: string; headers: Record<string, string>; body?: string },
) => Promise<{ status: number; text(): Promise<string> }>;
/** Hands out a current access token, refreshing it when expired or when forced (after a 401). */
export type SpotifyTokenSource = {
accessToken(forceRefresh: boolean): Promise<string>;
};
/** A failed request; `reason` is Spotify's player error code (NO_ACTIVE_DEVICE, PREMIUM_REQUIRED, ...). */
export class SpotifyApiError extends Error {
constructor(
message: string,
readonly status: number,
readonly reason = "",
) {
super(message);
this.name = "SpotifyApiError";
}
}
export type SpotifyImage = { url: string; width: number; height: number };
export type SpotifyItem = {
uri: string;
type: "track" | "episode";
name: string;
/** Track artists, comma-joined; for an episode, its show. */
artists: string;
/** The track's album; for an episode, its show. "" when the API leaves it out (album track listings). */
album: string;
durationMs: number;
images: SpotifyImage[];
};
export type SpotifyDevice = {
id: string;
name: string;
/** "Smartphone", "Computer", "Speaker", ... */
type: string;
active: boolean;
/** Restricted devices accept no Web API commands. */
restricted: boolean;
/** -1 when the device reports none. */
volumePercent: number;
supportsVolume: boolean;
};
export type SpotifyRepeat = "off" | "context" | "track";
export type SpotifyPlayback = {
device: SpotifyDevice | null;
playing: boolean;
progressMs: number;
shuffle: boolean;
repeat: SpotifyRepeat;
/** The playlist / album / artist playing from; "" for none (a queue of single tracks). */
contextUri: string;
/** null during an ad, between items, or for content the API does not describe. */
item: SpotifyItem | null;
/** "track", "episode", "ad" or "unknown". */
playingType: string;
/** The player actions the context forbids, as Spotify names them: pausing, resuming, skipping_next, ... */
disallows: string[];
};
export type SpotifyQueue = { current: SpotifyItem | null; upcoming: SpotifyItem[] };
export type SpotifyUser = { id: string; displayName: string; product: string };
export type SpotifyPlaylist = {
id: string;
uri: string;
name: string;
ownerId: string;
ownerName: string;
collaborative: boolean;
total: number;
};
export type SpotifyAlbum = { id: string; uri: string; name: string; artists: string };
export type SpotifyArtist = { id: string; uri: string; name: string };
/** A played track and the context it played from ("" when none). */
export type SpotifyPlayed = { item: SpotifyItem; contextUri: string };
export type SpotifySearchResults = {
tracks: SpotifyItem[];
albums: SpotifyAlbum[];
artists: SpotifyArtist[];
playlists: SpotifyPlaylist[];
};
export type SpotifySearchType = "track" | "album" | "artist" | "playlist";
/**
* What to play: a context (playlist, album, artist, show) optionally starting
* at one of its items, or a list of track URIs. Empty resumes the current
* playback.
*/
export type SpotifyPlayRequest = {
contextUri?: string;
uris?: string[];
offsetUri?: string;
offsetPosition?: number;
};
type Query = Record<string, string | number | boolean | undefined>;
export class SpotifyClient {
constructor(
private readonly fetchImpl: SpotifyFetch,
private readonly tokens: SpotifyTokenSource,
) {}
/** The current playback on whichever device is active; null when nothing is. */
async playback(): Promise<SpotifyPlayback | null> {
const raw = await this.request("GET", "/me/player", { query: { additional_types: "episode" } });
return raw ? parsePlayback(raw) : null;
}
async queue(): Promise<SpotifyQueue> {
const raw = await this.request("GET", "/me/player/queue");
return {
current: parseItem(raw?.currently_playing),
upcoming: asArray(raw?.queue).map(parseItem).filter(isPresent),
};
}
async devices(): Promise<SpotifyDevice[]> {
const raw = await this.request("GET", "/me/player/devices");
return asArray(raw?.devices).map(parseDevice).filter(isPresent);
}
async play(request: SpotifyPlayRequest = {}, deviceId = ""): Promise<void> {
const body: Record<string, unknown> = {};
if (request.contextUri) body.context_uri = request.contextUri;
if (request.uris?.length) body.uris = request.uris;
if (request.offsetUri) body.offset = { uri: request.offsetUri };
else if (request.offsetPosition !== undefined) body.offset = { position: request.offsetPosition };
await this.request("PUT", "/me/player/play", { query: { device_id: deviceId || undefined }, body });
}
async pause(): Promise<void> {
await this.request("PUT", "/me/player/pause");
}
async next(): Promise<void> {
await this.request("POST", "/me/player/next");
}
async previous(): Promise<void> {
await this.request("POST", "/me/player/previous");
}
async setShuffle(on: boolean): Promise<void> {
await this.request("PUT", "/me/player/shuffle", { query: { state: on } });
}
async setRepeat(mode: SpotifyRepeat): Promise<void> {
await this.request("PUT", "/me/player/repeat", { query: { state: mode } });
}
async setVolume(percent: number, deviceId = ""): Promise<void> {
const volume = Math.max(0, Math.min(100, Math.round(percent)));
await this.request("PUT", "/me/player/volume", { query: { volume_percent: volume, device_id: deviceId || undefined } });
}
/** Move playback to a device (one that shows in devices() but is not active), optionally playing. */
async transfer(deviceId: string, play: boolean): Promise<void> {
await this.request("PUT", "/me/player", { body: { device_ids: [deviceId], play } });
}
async me(): Promise<SpotifyUser> {
const raw = await this.request("GET", "/me");
return { id: str(raw?.id), displayName: str(raw?.display_name) || str(raw?.id), product: str(raw?.product) };
}
/** Whether each URI is in the user's library (Liked Songs for tracks). */
async libraryContains(uris: string[]): Promise<boolean[]> {
const raw = await this.request("GET", "/me/library/contains", { query: { uris: uris.join(",") } });
return asArray(raw).map(Boolean);
}
async saveToLibrary(uris: string[]): Promise<void> {
await this.request("PUT", "/me/library", { query: { uris: uris.join(",") } });
}
async removeFromLibrary(uris: string[]): Promise<void> {
await this.request("DELETE", "/me/library", { query: { uris: uris.join(",") } });
}
playlists(): Promise<SpotifyPlaylist[]> {
return this.collect("/me/playlists", { limit: 50 }, (raw) => asArray(raw?.items).map(parsePlaylist));
}
/** Only playlists the user owns or collaborates on list their items (403 otherwise). */
playlistItems(playlistId: string): Promise<SpotifyItem[]> {
return this.collect(`/playlists/${encodeURIComponent(playlistId)}/items`, { limit: 50, additional_types: "track,episode" },
(raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.item ?? entry?.track)));
}
savedTracks(): Promise<SpotifyItem[]> {
return this.collect("/me/tracks", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.track)));
}
savedAlbums(): Promise<SpotifyAlbum[]> {
return this.collect("/me/albums", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseAlbum(entry?.album)));
}
albumTracks(albumId: string): Promise<SpotifyItem[]> {
return this.collect(`/albums/${encodeURIComponent(albumId)}/tracks`, { limit: 50 }, (raw) => asArray(raw?.items).map(parseItem));
}
followedArtists(): Promise<SpotifyArtist[]> {
// Cursor-paged, with the page wrapped in an "artists" object.
return this.collect("/me/following", { type: "artist", limit: 50 }, (raw) => asArray(raw?.artists?.items).map(parseArtist),
(raw) => str(raw?.artists?.next));
}
artistAlbums(artistId: string): Promise<SpotifyAlbum[]> {
return this.collect(`/artists/${encodeURIComponent(artistId)}/albums`, { include_groups: "album,single,compilation", limit: 10 },
(raw) => asArray(raw?.items).map(parseAlbum), undefined, 100);
}
/** Recently played tracks, newest first (one page: the API keeps only the last 50). */
async recentlyPlayed(): Promise<SpotifyPlayed[]> {
const raw = await this.request("GET", "/me/player/recently-played", { query: { limit: 50 } });
return asArray(raw?.items)
.map((entry: any) => {
const item = parseItem(entry?.track);
return item ? { item, contextUri: str(entry?.context?.uri) } : null;
})
.filter(isPresent);
}
async search(query: string, types: SpotifySearchType[], limit = 5): Promise<SpotifySearchResults> {
const raw = await this.request("GET", "/search", {
query: { q: query, type: types.join(","), limit: Math.max(1, Math.min(10, Math.round(limit))) },
});
return {
tracks: asArray(raw?.tracks?.items).map(parseItem).filter(isPresent),
albums: asArray(raw?.albums?.items).map(parseAlbum).filter(isPresent),
artists: asArray(raw?.artists?.items).map(parseArtist).filter(isPresent),
// Search results include nulls for playlists the API can no longer return.
playlists: asArray(raw?.playlists?.items).map(parsePlaylist).filter(isPresent),
};
}
/** Gather a paged listing by following its `next` links, up to maxItems. */
private async collect<T>(
path: string,
query: Query,
parse: (raw: any) => Array<T | null>,
nextOf: (raw: any) => string = (raw) => str(raw?.next),
maxItems = MAX_LIST_ITEMS,
): Promise<T[]> {
const items: T[] = [];
let raw = await this.request("GET", path, { query });
for (;;) {
for (const item of parse(raw)) if (item) items.push(item);
const next = nextOf(raw);
if (!next || items.length >= maxItems) return items.slice(0, maxItems);
raw = await this.request("GET", next);
}
}
/**
* One API call. `path` is relative to the API base, or an absolute `next`
* link from a page. Resolves to the parsed JSON body (null when empty);
* a 401 refreshes the token and retries once.
*/
private async request(method: string, path: string, options: { query?: Query; body?: unknown } = {}): Promise<any> {
const url = path.startsWith("https://") ? path : `${SPOTIFY_API_BASE}${path}${buildQuery(options.query)}`;
const headers: Record<string, string> = { Accept: "application/json" };
if (options.body !== undefined) headers["Content-Type"] = "application/json";
// A command without a body still sends an empty one: Spotify answers a
// PUT or POST with no Content-Length with 411.
const body = options.body !== undefined ? JSON.stringify(options.body) : method === "GET" ? undefined : "";
for (let attempt = 0; ; attempt++) {
const authorization = `Bearer ${await this.tokens.accessToken(attempt > 0)}`;
const response = await withTimeout(this.fetchImpl(url, { method, headers: { ...headers, Authorization: authorization }, body }));
const text = await withTimeout(response.text());
if (response.status === 401 && attempt === 0) continue;
if (response.status >= 200 && response.status < 300) {
if (!text.trim()) return null;
try {
return JSON.parse(text);
} catch {
// Commands answer 200 with a snapshot id or plain text; nothing to read.
return null;
}
}
throw apiError(response.status, text);
}
}
}
/** Form / query encoding (not URLSearchParams, which the NativeScript runtime mishandles). */
export function buildQuery(query: Query | undefined): string {
if (!query) return "";
const parts = Object.keys(query)
.filter((key) => query[key] !== undefined && query[key] !== "")
.map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(String(query[key]))}`);
return parts.length ? `?${parts.join("&")}` : "";
}
async function withTimeout<T>(promise: Promise<T>): Promise<T> {
let timer: ReturnType<typeof setTimeout> | null = null;
const timeout = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new SpotifyApiError("Spotify didn't answer. Check the phone's connection.", 0)), HTTP_TIMEOUT_MS);
});
try {
return await Promise.race([promise, timeout]);
} catch (error) {
if (error instanceof SpotifyApiError) throw error;
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
} finally {
if (timer) clearTimeout(timer);
}
}
/** A message for the glasses from Spotify's error body ({error: {status, message, reason}}). */
function apiError(status: number, body: string): SpotifyApiError {
let message = "";
let reason = "";
try {
const parsed = JSON.parse(body);
message = str(parsed?.error?.message ?? parsed?.error_description);
reason = str(parsed?.error?.reason);
} catch {
// Not JSON (a gateway error page).
}
switch (reason) {
case "NO_ACTIVE_DEVICE":
return new SpotifyApiError("Spotify isn't open on any device. Open Spotify on your phone, then try again.", status, reason);
case "PREMIUM_REQUIRED":
return new SpotifyApiError("Controlling Spotify playback needs Spotify Premium.", status, reason);
}
if (status === 401) return new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, reason);
if (status === 403 && /registered|developer dashboard/i.test(message)) {
return new SpotifyApiError("This Spotify account isn't on the client ID's user list (Developer Dashboard > User Management).", status, reason);
}
if (status === 429) return new SpotifyApiError("Spotify is limiting requests; try again in a minute.", status, reason);
return new SpotifyApiError(message ? `Spotify: ${message}` : `Spotify error (HTTP ${status}).`, status, reason);
}
export function parsePlayback(raw: any): SpotifyPlayback {
const repeat = str(raw?.repeat_state);
const disallows = raw?.actions?.disallows;
return {
device: parseDevice(raw?.device),
playing: raw?.is_playing === true,
progressMs: num(raw?.progress_ms, 0),
shuffle: raw?.shuffle_state === true,
repeat: repeat === "context" || repeat === "track" ? repeat : "off",
contextUri: str(raw?.context?.uri),
item: parseItem(raw?.item),
playingType: str(raw?.currently_playing_type) || "unknown",
disallows: disallows && typeof disallows === "object" ? Object.keys(disallows).filter((key) => disallows[key] === true) : [],
};
}
export function parseItem(raw: any): SpotifyItem | null {
if (!raw || typeof raw !== "object" || typeof raw.uri !== "string" || !raw.uri) return null;
if (raw.type === "episode") {
const show = str(raw.show?.name);
return {
uri: raw.uri,
type: "episode",
name: str(raw.name),
artists: show || str(raw.show?.publisher),
album: show,
durationMs: num(raw.duration_ms, -1),
images: parseImages(raw.images?.length ? raw.images : raw.show?.images),
};
}
return {
uri: raw.uri,
type: "track",
name: str(raw.name),
artists: joinNames(raw.artists),
album: str(raw.album?.name),
durationMs: num(raw.duration_ms, -1),
images: parseImages(raw.album?.images),
};
}
export function parseDevice(raw: any): SpotifyDevice | null {
if (!raw || typeof raw !== "object") return null;
const volume = raw.volume_percent;
return {
id: str(raw.id),
name: str(raw.name),
type: str(raw.type),
active: raw.is_active === true,
restricted: raw.is_restricted === true,
volumePercent: typeof volume === "number" && Number.isFinite(volume) ? volume : -1,
supportsVolume: raw.supports_volume === true,
};
}
function parsePlaylist(raw: any): SpotifyPlaylist | null {
if (!raw || typeof raw !== "object" || !raw.id) return null;
return {
id: str(raw.id),
uri: str(raw.uri) || `spotify:playlist:${raw.id}`,
name: str(raw.name),
ownerId: str(raw.owner?.id),
ownerName: str(raw.owner?.display_name) || str(raw.owner?.id),
collaborative: raw.collaborative === true,
// `items` replaced the deprecated `tracks` summary in 2026.
total: num(raw.items?.total ?? raw.tracks?.total, -1),
};
}
function parseAlbum(raw: any): SpotifyAlbum | null {
if (!raw || typeof raw !== "object" || !raw.id) return null;
return { id: str(raw.id), uri: str(raw.uri) || `spotify:album:${raw.id}`, name: str(raw.name), artists: joinNames(raw.artists) };
}
function parseArtist(raw: any): SpotifyArtist | null {
if (!raw || typeof raw !== "object" || !raw.id) return null;
return { id: str(raw.id), uri: str(raw.uri) || `spotify:artist:${raw.id}`, name: str(raw.name) };
}
function parseImages(raw: any): SpotifyImage[] {
return asArray(raw)
.filter((image: any) => image && typeof image.url === "string" && image.url)
.map((image: any) => ({ url: image.url, width: num(image.width, 0), height: num(image.height, 0) }));
}
function joinNames(raw: any): string {
return asArray(raw).map((entry: any) => str(entry?.name)).filter(Boolean).join(", ");
}
function asArray(value: unknown): any[] {
return Array.isArray(value) ? value : [];
}
function str(value: unknown): string {
return typeof value === "string" ? value : "";
}
function num(value: unknown, fallback: number): number {
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
}
function isPresent<T>(value: T | null | undefined): value is T {
return value !== null && value !== undefined;
}
+65
View File
@@ -0,0 +1,65 @@
import { fetchTextWithUserAgent } from "../util/http";
import { downloadBinary } from "./binary-http.ios";
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
import { fromData, toData } from "./kotlin-data";
import type { WebSignIn } from "./spotify-auth";
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
import { SpotifyService } from "./spotify-service";
declare const FaceclawAppleMusic: any;
declare const FaceclawCrypto: any;
declare const FaceclawGraphics: any;
declare const FaceclawKitIosProtocol: any;
declare const FaceclawWebSignIn: any;
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
/** Sign-in in the system browser sheet, which FaceclawWebSignIn closes once the loopback redirect arrives. */
const webSignIn: WebSignIn = {
createPkce: () => {
// 64 URL-safe characters (RFC 7636 allows 43..128).
const verifier = String(FaceclawCrypto.randomBase64Url(48));
return { verifier, challenge: String(FaceclawCrypto.sha256Base64Url(verifier)) };
},
randomState: () => String(FaceclawCrypto.randomBase64Url(16)),
open: (authorizeUrl, port, path) =>
new Promise<string>((resolve, reject) => {
FaceclawWebSignIn.shared().signInPortPathCompletion(authorizeUrl, port, path, (redirectUrl: string | null, error: string | null) => {
if (redirectUrl) resolve(String(redirectUrl));
else reject(new Error(String(error || "Sign-in failed")));
});
}),
cancel: () => FaceclawWebSignIn.shared().cancel(),
};
let protocol: any = null;
let service: SpotifyService | null = null;
/** The Spotify service, created on first use (main thread only: it owns timers and the sign-in sheet). */
export function spotifyService(): SpotifyService | null {
service ??= new SpotifyService({
packageName: SPOTIFY_IOS_BUNDLE_ID,
fetch: (url, init) => fetchTextWithUserAgent(url, init),
webSignIn,
download: async (url) => {
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
return response.status === 200 ? response.bytes : null;
},
decodeArt: (bytes, maxSize) => {
const packet = FaceclawGraphics.bgraPacketFromImageDataMaxSize(toData(bytes), Math.round(maxSize));
if (!packet) return null;
// The shared Kotlin tone curve and dither, as for Apple Music's art.
protocol ??= FaceclawKitIosProtocol.new();
return grayImageFromPacket(fromData(protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither)));
},
// FaceclawAppleMusic's system output volume needs no media-library access.
systemVolume: {
get: () => {
const volume = Number(FaceclawAppleMusic.shared().volumePercent());
return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1;
},
set: (percent) => FaceclawAppleMusic.shared().setVolumePercent(Math.max(0, Math.min(100, Math.round(percent)))),
},
});
return service;
}
+124
View File
@@ -0,0 +1,124 @@
import { Application, Utils } from "@nativescript/core";
import { fetchTextWithUserAgent } from "../util/http";
import { downloadBinary } from "./binary-http";
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
import { copyToJavaByteBuffer } from "./java-direct-buffer";
import type { WebSignIn } from "./spotify-auth";
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
import { SpotifyService } from "./spotify-service";
declare const android: any;
declare const com: any;
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
/** The sign-in's listener proxy, held while Java may still call it. */
let signInListener: unknown = null;
/** Faceclaw's activity is in front, so it may open the browser (Android blocks background activity starts). */
function foregroundActivity(): any {
return Application.suspended ? null : Application.android?.foregroundActivity ?? null;
}
/**
* Sign-in in the phone's browser, with FaceclawWebSignIn catching the
* loopback redirect. A sign-in started from the glasses while Faceclaw is in
* the background opens the page when Faceclaw next comes to the front, as on
* iOS.
*/
const webSignIn: WebSignIn = {
createPkce: () => {
// 64 URL-safe characters (RFC 7636 allows 43..128).
const verifier = String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(48));
return { verifier, challenge: String(com.faceclaw.app.FaceclawWebSignIn.sha256Base64Url(verifier)) };
},
randomState: () => String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(16)),
open: (authorizeUrl, port, path) =>
new Promise<string>((resolve, reject) => {
const native = com.faceclaw.app.FaceclawWebSignIn.shared();
let onResume: ((args: any) => void) | null = null;
const stopWaiting = () => {
if (onResume) Application.off(Application.resumeEvent, onResume);
onResume = null;
};
const listener = new com.faceclaw.app.FaceclawWebSignInListener({
onFinished: (redirectUrl: string | null, error: string | null) => {
if (signInListener === listener) signInListener = null;
stopWaiting();
if (redirectUrl) resolve(String(redirectUrl));
else reject(new Error(String(error || "Sign-in failed")));
},
});
const startError = native.start(port, path, listener);
if (startError) {
reject(new Error(String(startError)));
return;
}
signInListener = listener;
const present = (activity: any) => {
stopWaiting();
const openError = native.openBrowser(activity, authorizeUrl);
if (openError) native.fail(String(openError));
};
const activity = foregroundActivity();
if (activity) {
present(activity);
return;
}
onResume = (args: any) => present(args?.activity ?? args?.android ?? Utils.android.getApplicationContext());
Application.on(Application.resumeEvent, onResume);
}),
cancel: () => com.faceclaw.app.FaceclawWebSignIn.shared().cancel(),
};
/** The phone's media-stream volume, for Spotify playing on this phone (as the session bridge reports it). */
function audioManager(): any {
return Utils.android.getApplicationContext()?.getSystemService(android.content.Context.AUDIO_SERVICE) ?? null;
}
const systemVolume = {
get: (): number => {
const audio = audioManager();
if (!audio) return -1;
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
if (max <= 0) return 0;
return Math.max(0, Math.min(100, Math.round((Number(audio.getStreamVolume(android.media.AudioManager.STREAM_MUSIC)) * 100) / max)));
},
set: (percent: number): void => {
const audio = audioManager();
if (!audio) return;
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
const volume = Math.round((Math.max(0, Math.min(100, percent)) * max) / 100);
audio.setStreamVolume(android.media.AudioManager.STREAM_MUSIC, volume, 0);
},
};
let service: SpotifyService | null = null;
/**
* The Spotify Web API service, created on first use (main thread only: it
* owns timers and the sign-in). Signing in is optional on Android: Spotify's
* media session already gives now-playing and transport control, and the
* media bridges add the Web API's extras once signed in (library browsing,
* search, shuffle / repeat / like, playback on other Spotify devices).
*/
export function spotifyService(): SpotifyService | null {
service ??= new SpotifyService({
packageName: SPOTIFY_ANDROID_PACKAGE,
fetch: (url, init) => fetchTextWithUserAgent(url, init),
webSignIn,
download: async (url) => {
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
return response.status === 200 ? response.bytes : null;
},
decodeArt: (bytes, maxSize) => {
const size = Math.round(maxSize);
// The shared photo tone (gamma + dither), as for media-session art.
return grayImageFromPacket(
com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(copyToJavaByteBuffer(bytes), size, size, PHOTO_TONE.gamma, PHOTO_TONE.dither),
);
},
systemVolume,
});
return service;
}
+159 -16
View File
@@ -2,7 +2,9 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
import { WavRecorder } from './wav-recorder.ios'
import { pcmLevel } from '../util/audio-level'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
@@ -10,20 +12,26 @@ export type IosMicrophoneSession = {
}
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
const CLOUD_FINAL_TIMEOUT_MS = 5000
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
const RAW_RETRY_MS = 3000
const RAW_RETRY_LIMIT = 5
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments. When the
* session or the app lifecycle cuts one short it is drained, not discarded;
* restarting it is the controller's call.
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
* is open-ended and has pauses commit its segments, so it needs a cloud
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
* recognizer only handles short dictations. When the session or the app
* lifecycle cuts one short it is drained, not discarded; restarting it is the
* controller's call.
*/
export class IosVoiceControlBridge {
private native: any = null
private readonly statuses = new Set<(state: VoiceControlState) => void>()
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
private readonly ends = new Set<() => void>()
private readonly levels = new Set<(level: number) => void>()
private status = 'Voice input ready.'
private generation = 0
private session: IosMicrophoneSession | null = null
@@ -35,10 +43,13 @@ export class IosVoiceControlBridge {
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
// on-device continuous capture) owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: ReconnectingSttClient | null = null
// What transcribes the current capture, for the log.
private recognition = 'on-device'
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
@@ -51,6 +62,21 @@ export class IosVoiceControlBridge {
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
// rawListeners while any owner wants it, from the session setRawSession
// bound. It shares the one glasses mic with speech capture, which preempts
// it; it resumes on its own once that capture has ended.
private rawNative: any = null
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
private readonly rawOwners = new Set<string>()
private rawSession: IosMicrophoneSession | null = null
// The session the tap enabled the mic on, while it is starting or live.
private rawHeld: IosMicrophoneSession | null = null
private rawLive = false
private rawGeneration = 0
private rawFailures = 0
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
private rawLog: (message: string) => void = () => {}
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
@@ -60,15 +86,19 @@ export class IosVoiceControlBridge {
}
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
onSpeechPause(_listener: () => void): () => void { return () => {} }
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
private ensureNative(): any {
if (!this.native) {
this.native = FaceclawSpeech.new()
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
this.native.pcmHandler = (pcm: NSData) => {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
// Copied: bufferFromData doesn't retain pcm, and STT clients queue chunks.
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
this.cloud?.acceptPcm(bytes)
if (this.continuous) this.record(bytes)
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
}
}
return this.native
@@ -96,6 +126,8 @@ export class IosVoiceControlBridge {
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
this.endRawTap()
this.stop()
const generation = ++this.generation
this.continuous = continuous
@@ -111,7 +143,7 @@ export class IosVoiceControlBridge {
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
@@ -125,24 +157,34 @@ export class IosVoiceControlBridge {
}
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.endRawTap()
this.stop()
this.continuous = continuous
this.log = log; this.source = 'phone'; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = this.startNative(voice, endpointing, true)
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
}
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
const native = this.ensureNative()
const generation = this.generation
const current = () => generation === this.generation
const cloud = createCloudSttClient(voice, {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => this.continuous)
const options = {
language: voice.language,
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
onStatus: (status: string) => { if (current()) this.setStatus(status) },
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}
const continuous = () => this.continuous
let cloud = createCloudSttClient(voice, options, continuous)
this.recognition = cloud ? 'cloud' : 'on-device'
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
if (!cloud && this.continuous && speechAnalyzerSupported()) {
cloud = createSpeechAnalyzerClient(options, continuous)
this.recognition = 'on-device (SpeechAnalyzer)'
}
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
@@ -155,8 +197,11 @@ export class IosVoiceControlBridge {
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
/**
* Whether continuous capture can run with this provider: a cloud one, or
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
*/
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
@@ -222,6 +267,100 @@ export class IosVoiceControlBridge {
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
this.stop(); this.setStatus(message); this.emitEnd()
}
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
}
/**
* The glasses session the raw tap captures from (connected and unlocked),
* or null. Owners outlive a change: the tap moves to the next session.
*/
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
this.rawLog = log
if (session === this.rawSession) return
this.endRawTap()
this.rawSession = session; this.rawFailures = 0
this.syncRawTap()
}
/**
* Start the decode-only raw tap on behalf of `owner`; the communicator is
* the one setRawSession bound, not `options.communicator`. False while
* speech capture holds the mic or no session is bound: the owner stays
* registered and the tap starts once it can. Idempotent while running.
*/
startRawCapture(options: { communicator: any; owner: string }): boolean {
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
this.syncRawTap()
return this.rawHeld !== null
}
/** `owner` no longer wants the raw tap; it stops once no owner does. */
stopRawCapture(owner: string): void {
if (this.rawOwners.delete(owner)) this.syncRawTap()
}
/** Whether the raw tap's glasses mic enable went through and still holds. */
isRawCaptureLive(): boolean { return this.rawLive }
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
private syncRawTap(): void {
const session = this.rawSession
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
if (this.rawHeld || this.rawRetryTimer !== null) return
void this.beginRawTap(session)
}
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
const generation = ++this.rawGeneration
const native = this.ensureRawNative()
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
if (error) { this.failRawTap(error); return }
this.rawHeld = session
try {
await session.setMicrophone(true, packet => {
if (generation !== this.rawGeneration) return
const copy = new Uint8Array(packet)
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.rawGeneration) return
this.rawLive = true
this.rawLog('EvenHub mic: glasses microphone enabled')
} catch (failure) {
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
}
}
private ensureRawNative(): any {
if (!this.rawNative) {
this.rawNative = FaceclawSpeech.new()
// A PCM capture only ends on its own when decoding failed or fell behind.
this.rawNative.eventHandler = (json: string) => {
const event = JSON.parse(json)
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
}
this.rawNative.pcmHandler = (pcm: NSData) => {
// Copied: bufferFromData doesn't retain pcm.
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
// Audio is flowing, so a later failure is a new problem worth retrying.
this.rawFailures = 0
for (const fn of [...this.rawListeners]) fn(bytes)
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
}
}
return this.rawNative
}
/** Drop the tap and retry it shortly, a bounded number of times. */
private failRawTap(message: string): void {
this.rawLog(`EvenHub mic: ${message}`)
this.endRawTap()
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
}
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
private endRawTap(): void {
++this.rawGeneration
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
this.rawRetryTimer = null
this.rawLive = false
this.rawNative?.cancel()
const session = this.rawHeld; this.rawHeld = null
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
}
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
private releaseMicrophone(): void {
const session = this.session; this.session = null
@@ -255,7 +394,11 @@ export class IosVoiceControlBridge {
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private emitEnd(): void {
for (const fn of [...this.ends]) fn()
// After the listeners: one may start the next capture, which keeps the mic.
this.syncRawTap()
}
private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
}
+18 -1
View File
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
declare const com: any;
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
// stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private readonly micLevelListeners = new Set<(level: number) => void>();
private rawActive = false;
// Who wants the raw tap. It runs while any owner does, so one owner letting
// go (an EvenHub app backgrounding) doesn't silence another.
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
return () => this.rawPcmListeners.delete(listener);
}
/**
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
* each decoded audio chunk, about every 50 ms, from any capture: STT in
* every mode, or the raw tap.
*/
onMicLevel(listener: (level: number) => void): () => void {
this.micLevelListeners.add(listener);
return () => this.micLevelListeners.delete(listener);
}
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
onFrameMeta(listener: FrameMetaListener): () => void {
this.frameMetaListeners.add(listener);
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
if (this.micLevelListeners.size > 0) {
const level = pcmLevel(bytes);
this.micLevelListeners.forEach((listener) => listener(level));
}
},
onFrameMeta: (angleDegrees: number, ssr: number) => {
if (this.frameMetaListeners.size === 0) return;