forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes
This commit is contained in:
commit
fcea0b369a
218 files changed
+14438
-2259
No files matched your search
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Platform-neutral ambient-light types and the CFW report decoder (settings
|
||||
* field 105), shared by ambient-light.ts and ambient-light.ios.ts.
|
||||
*/
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
|
||||
/** The options with their defaults filled in. */
|
||||
export function resolvePollingOptions(options: AmbientLightPollingOptions): Required<AmbientLightPollingOptions> {
|
||||
return {
|
||||
intervalMs: options.intervalMs ?? 500,
|
||||
minDelta: options.minDelta ?? 1,
|
||||
heartbeatMs: options.heartbeatMs ?? 5000,
|
||||
bindToLease: options.bindToLease ?? true,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
/**
|
||||
* Ambient light sensor access on iOS: the same API as ambient-light.ts (see
|
||||
* there for the CFW query and passive-polling semantics) over the running
|
||||
* shared-Kotlin session. No-ops while no session is running.
|
||||
*/
|
||||
|
||||
/** Ask the firmware for one ambient-light report (delivered to listeners). */
|
||||
export function queryAmbientLight(): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.queryAmbientLight();
|
||||
} catch (error) {
|
||||
console.warn(`queryAmbientLight failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Start or stop CFW passive light-sensor polling. */
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
|
||||
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addAmbientLightListener((body) => {
|
||||
const report = decodeAmbientLightReport(body);
|
||||
if (report) listener(report);
|
||||
});
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -16,88 +16,17 @@
|
||||
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
|
||||
*/
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -123,14 +52,9 @@ export function queryAmbientLight(): void {
|
||||
* on every session start.
|
||||
*/
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
activeCommunicator()?.setAmbientLightPolling(
|
||||
enabled,
|
||||
options.intervalMs ?? 500,
|
||||
options.minDelta ?? 1,
|
||||
options.heartbeatMs ?? 5000,
|
||||
options.bindToLease ?? true,
|
||||
);
|
||||
activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/**
|
||||
* Passive view of the glasses' audio traffic on iOS: the same API as
|
||||
* audio-monitor.ts over the running shared-Kotlin session. Nothing here sends
|
||||
* anything to the glasses.
|
||||
*/
|
||||
|
||||
export type AudioArm = "L" | "R" | "?";
|
||||
|
||||
/** The session listeners attach to; changes across reconnects. */
|
||||
export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */
|
||||
export function isFaceclawAudioCaptureActive(): boolean {
|
||||
try {
|
||||
return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe `communicator` to every audio-characteristic packet and every
|
||||
* field-104 mic status record body (undecoded). Delivered on the main thread.
|
||||
* Returns an unsubscribe function.
|
||||
*/
|
||||
export function addAudioMonitorListeners(
|
||||
communicator: FaceclawCommunicatorBridge | null,
|
||||
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
|
||||
onStatus: (body: Uint8Array, arm: AudioArm) => void,
|
||||
): () => void {
|
||||
if (!communicator) return () => {};
|
||||
const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs));
|
||||
const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm)));
|
||||
return () => {
|
||||
try {
|
||||
offPackets();
|
||||
offStatus();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function toArm(arm: string): AudioArm {
|
||||
return arm === "L" || arm === "R" ? arm : "?";
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
export type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
|
||||
// The iOS twin of the Android benchmark bridge. iOS has no static active
|
||||
// communicator, so the preview controller binds its session here on connect
|
||||
// and unbinds it on close; the run itself is the same shared-Kotlin
|
||||
// GlassesSessionCore stream as on Android.
|
||||
type BenchmarkSession = {
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean
|
||||
cancelBandwidthBenchmark(): void
|
||||
getBandwidthBenchmarkStatus(): string
|
||||
}
|
||||
let session: BenchmarkSession | null = null
|
||||
|
||||
export function bindBandwidthBenchmarkSession(next: BenchmarkSession | null): void {
|
||||
session = next
|
||||
}
|
||||
|
||||
/** Start a run. False if not connected or a run is already active. */
|
||||
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
|
||||
if (!session) return false
|
||||
try {
|
||||
return session.startBandwidthBenchmark(
|
||||
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode))
|
||||
} catch (error) {
|
||||
console.warn(`startBandwidthBenchmark failed: ${error}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Cancel an in-progress run. Safe to call when idle. */
|
||||
export function cancelBandwidthBenchmark(): void {
|
||||
try {
|
||||
session?.cancelBandwidthBenchmark()
|
||||
} catch (error) {
|
||||
console.warn(`cancelBandwidthBenchmark failed: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Status of the current or most recent run; null when not connected. */
|
||||
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
|
||||
if (!session) return null
|
||||
try {
|
||||
return JSON.parse(session.getBandwidthBenchmarkStatus()) as BandwidthBenchmarkStatus
|
||||
} catch (error) {
|
||||
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,9 @@ export type BandwidthBenchmarkStatus = {
|
||||
payloadBytesAcked: number;
|
||||
wireBytesAcked: number;
|
||||
aborted: boolean;
|
||||
/** Each arm's current PHY ("2M", "1M", ...); null when the platform doesn't report it. */
|
||||
leftPhy?: string | null;
|
||||
rightPhy?: string | null;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
|
||||
@@ -2,11 +2,18 @@ import { Http } from '@nativescript/core';
|
||||
import { withUserAgent } from '../util/http';
|
||||
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
|
||||
|
||||
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
|
||||
/**
|
||||
* Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies.
|
||||
*
|
||||
* toArrayBuffer() is interop.bufferFromData, which in @nativescript/ios 9.0.3
|
||||
* views the NSData's bytes without retaining it: once the response is
|
||||
* collected, the buffer is dangling (a store EHPK was saved as 41 MB of
|
||||
* zeros). Copy while the response is still alive.
|
||||
*/
|
||||
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
|
||||
const response = await Http.request({ url, method: 'GET', headers: withUserAgent(), timeout: timeoutMs });
|
||||
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()).slice(), contentLength: Number(length ?? 0) };
|
||||
}
|
||||
|
||||
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
|
||||
|
||||
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
};
|
||||
|
||||
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
create: (options, selection) => new WhisperServerSttClient(options, {
|
||||
baseUrl: selection.whisperServerUrl ?? "",
|
||||
model: selection.whisperServerModel ?? "",
|
||||
language: selection.whisperServerLanguage ?? "",
|
||||
language: selection.language ?? "",
|
||||
}),
|
||||
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
|
||||
},
|
||||
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture transcribes with: the provider, and for Whisper server the
|
||||
* language it is told to expect. A running capture is shared only by holders
|
||||
* asking for the same.
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
|
||||
return selection.provider === "whisper-server"
|
||||
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
|
||||
: selection.provider;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
@@ -105,7 +106,7 @@ export function createCloudSttClient(
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => provider.create(clientOptions, selection),
|
||||
{ ...options, apiKey },
|
||||
{ ...options, apiKey, language: (selection.language ?? "").trim() },
|
||||
continuous,
|
||||
provider.finalTimeoutMs,
|
||||
);
|
||||
|
||||
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
|
||||
|
||||
export type CloudSttOptions = {
|
||||
apiKey: string;
|
||||
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
|
||||
language?: string;
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
|
||||
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
// Without language_code the model detects the language, which can go
|
||||
// badly wrong on short or noisy segments (Japanese heard as Russian).
|
||||
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input";
|
||||
import { File, knownFolders, path, type ImageSource } from "@nativescript/core";
|
||||
import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat";
|
||||
import { type CompassEvent } from "./compass-types";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
import { fromData, toData } from "./kotlin-data";
|
||||
import { previewPixels } from "./ios-graphics";
|
||||
import { iosBluetooth } from "./ios-bluetooth";
|
||||
@@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any;
|
||||
declare const FaceclawKitIosAncsListener: any;
|
||||
declare const FaceclawKitIosAudioPacketListener: any;
|
||||
declare const FaceclawKitFaceclawCompassListener: any;
|
||||
declare const FaceclawKitFaceclawImuListener: any;
|
||||
declare const FaceclawKitIosAmbientLightListener: any;
|
||||
declare const FaceclawKitIosMicStatusListener: any;
|
||||
|
||||
|
||||
export type CommunicatorPhase =
|
||||
@@ -164,6 +168,29 @@ class CompassListener extends NSObject {
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class ImuListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitFaceclawImuListener];
|
||||
handler!: (reading: ImuReading) => void;
|
||||
onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void {
|
||||
this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) });
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class AmbientLightListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosAmbientLightListener];
|
||||
handler!: (body: Uint8Array) => void;
|
||||
onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); }
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class MicStatusListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosMicStatusListener];
|
||||
handler!: (body: Uint8Array, arm: string) => void;
|
||||
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS twin of the Android communicator bridge: the same public API over
|
||||
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
|
||||
@@ -173,6 +200,8 @@ class CompassListener extends NSObject {
|
||||
* Android so callers never re-enter the bridge from a Kotlin frame.
|
||||
*/
|
||||
export class FaceclawCommunicatorBridge {
|
||||
// The started, not yet closed bridge (see getActive).
|
||||
private static active: FaceclawCommunicatorBridge | null = null;
|
||||
private readonly communicator: any;
|
||||
private readonly listenerProxy: SessionListener;
|
||||
private ancsProxy: AncsListener | null = null;
|
||||
@@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge {
|
||||
private audioListener: AudioPacketListener | null = null;
|
||||
private microphoneWork: Promise<void> = Promise.resolve();
|
||||
private microphoneToken = 0;
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* The bridge whose session is running, for modules that reach the glasses
|
||||
* without the bridge threaded through (imu, ambient-light, audio-monitor):
|
||||
* the iOS stand-in for Android's FaceclawBleCommunicator.getActive().
|
||||
* Set by start(), cleared by close().
|
||||
*/
|
||||
static getActive(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.active;
|
||||
}
|
||||
|
||||
constructor(addresses: { right: string; left: string; ring?: string; ringRole?: string }) {
|
||||
this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress(
|
||||
@@ -454,7 +494,9 @@ export class FaceclawCommunicatorBridge {
|
||||
// ----- session ---------------------------------------------------------------------
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.start(); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this;
|
||||
});
|
||||
}
|
||||
|
||||
async setG2ScreenOn(screenOn: boolean): Promise<void> {
|
||||
@@ -644,6 +686,8 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.closed = true;
|
||||
if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null;
|
||||
this.setAncsListeners(false);
|
||||
this.previewAnimationListeners.clear();
|
||||
await this.enqueueNativeCall(() => {
|
||||
@@ -667,6 +711,85 @@ export class FaceclawCommunicatorBridge {
|
||||
return () => { this.communicator.removeCompassListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Developer > Debug tests > BLE bandwidth; see app/native/bandwidth-benchmark.ios.ts. */
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean {
|
||||
return Boolean(this.communicator.startBandwidthBenchmarkWithLinkModeMessageSizeWindowSizeDurationMsLinkMode(
|
||||
messageSize, windowSize, durationMs, linkMode));
|
||||
}
|
||||
|
||||
cancelBandwidthBenchmark(): void {
|
||||
this.communicator.cancelBandwidthBenchmark();
|
||||
}
|
||||
|
||||
/** Status/results of the current or most recent run, as JSON. */
|
||||
getBandwidthBenchmarkStatus(): string {
|
||||
return String(this.communicator.getBandwidthBenchmarkStatus());
|
||||
}
|
||||
|
||||
// ----- sensors and passive audio, for the developer demos -----------------------------
|
||||
// Fire-and-forget, so they skip the native-call queue like the compass controls do.
|
||||
|
||||
/** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */
|
||||
setImuReportEnabled(enable: boolean, reportFrq: number): void {
|
||||
this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq)));
|
||||
}
|
||||
|
||||
/** Accelerometer samples, delivered on the main queue. */
|
||||
addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const proxy = ImuListener.new() as ImuListener;
|
||||
proxy.handler = listener;
|
||||
this.communicator.addImuListenerListener(proxy);
|
||||
return () => { this.communicator.removeImuListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */
|
||||
queryAmbientLight(): void {
|
||||
this.communicator.queryAmbientLight();
|
||||
}
|
||||
|
||||
setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void {
|
||||
this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable),
|
||||
Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease));
|
||||
}
|
||||
|
||||
/** Raw field-105 ambient-light records, delivered on the main queue. */
|
||||
addAmbientLightListener(listener: (body: Uint8Array) => void): () => void {
|
||||
const proxy = AmbientLightListener.new() as AmbientLightListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAmbientLightListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */
|
||||
addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const proxy = MicStatusListener.new() as MicStatusListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addMicStatusListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeMicStatusListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */
|
||||
sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean {
|
||||
return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple(
|
||||
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every audio-characteristic packet, whatever enabled the mics, on the
|
||||
* main queue. Passive: sends nothing and doesn't affect capture.
|
||||
*/
|
||||
addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void {
|
||||
const proxy = AudioPacketListener.new() as AudioPacketListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAudioMonitorListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */
|
||||
isAudioCaptureActive(): boolean {
|
||||
return Boolean(this.communicator.isAudioCaptureActive());
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialized glasses microphone enable/disable (the voice bridge's
|
||||
* IosMicrophoneSession contract). A release while an enable awaits its ACK
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
/**
|
||||
* IMU (accelerometer) access on iOS: the same API as imu.ts over the running
|
||||
* shared-Kotlin session. As there, the stream is experimental, the calls are
|
||||
* no-ops while no session is running, and a listener stays with the session
|
||||
* it subscribed to.
|
||||
*/
|
||||
|
||||
/** Enable/disable the accelerometer report stream. No-op if not connected. */
|
||||
export function setImuReportEnabled(enable: boolean, reportFreq: number): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq);
|
||||
} catch (error) {
|
||||
console.warn(`setImuReportEnabled failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to IMU readings; returns an unsubscribe function. */
|
||||
export function addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addImuListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may be gone after a disconnect; nothing to clean up.
|
||||
}
|
||||
};
|
||||
}
|
||||
+3
-15
@@ -12,7 +12,9 @@ declare const com: any;
|
||||
* static, so it works from any isolate without threading the bridge through.
|
||||
*/
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||
|
||||
export type LocalModelState = {
|
||||
/**
|
||||
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||
*/
|
||||
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||
detail?: string;
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const FaceclawSystemLanguageModel: any;
|
||||
|
||||
/**
|
||||
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||
*
|
||||
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||
* system model runs out of process on the Neural Engine, needs no download,
|
||||
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||
* and tool definitions into it), Apple's guardrails, a background request
|
||||
* budget, and Apple Intelligence-capable hardware.
|
||||
*
|
||||
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||
* parks each call and reports it; here that ends the loop iteration with
|
||||
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||
* provider, and its next request (ending in the matching tool_result blocks)
|
||||
* resumes the parked generation instead of starting a new one.
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "apple-foundation-model",
|
||||
label: "Apple Intelligence",
|
||||
sizeBytes: 0,
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
/**
|
||||
* Only used when tool results arrive with no parked generation to resume (it
|
||||
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||
* continue after a tool output without a new prompt.
|
||||
*/
|
||||
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||
|
||||
type Availability =
|
||||
| "available"
|
||||
| "unsupportedOS"
|
||||
| "deviceNotEligible"
|
||||
| "appleIntelligenceNotEnabled"
|
||||
| "modelNotReady"
|
||||
| "unavailable";
|
||||
|
||||
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "needs iOS 26",
|
||||
deviceNotEligible: "not on this iPhone",
|
||||
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||
modelNotReady: "downloading",
|
||||
unavailable: "unavailable",
|
||||
};
|
||||
|
||||
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||
appleIntelligenceNotEnabled:
|
||||
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||
};
|
||||
|
||||
export function localModelAvailability(): Availability {
|
||||
try {
|
||||
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||
const value = String(FaceclawSystemLanguageModel.availability());
|
||||
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||
} catch {
|
||||
return "unsupportedOS";
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
const availability = localModelAvailability();
|
||||
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
return localModelAvailability() === "available";
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
const availability = localModelAvailability();
|
||||
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||
}
|
||||
|
||||
/** The user-facing reason the model can't run, or "" when it can. */
|
||||
export function localModelUnavailableMessage(): string {
|
||||
const availability = localModelAvailability();
|
||||
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||
}
|
||||
|
||||
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||
// delete or unload, and no progress to watch.
|
||||
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||
return () => {};
|
||||
}
|
||||
export function startLocalModelDownload(): void {}
|
||||
export function cancelLocalModelDownload(): void {}
|
||||
export function deleteLocalModel(): void {}
|
||||
export function unloadLocalModel(): void {}
|
||||
|
||||
type HistoryEntry =
|
||||
| { type: "prompt" | "response"; text: string }
|
||||
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||
|
||||
export type SystemModelRequest = {
|
||||
system: string;
|
||||
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||
history: HistoryEntry[];
|
||||
prompt: string;
|
||||
maxTokens: number;
|
||||
permissive: boolean;
|
||||
};
|
||||
|
||||
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||
|
||||
/**
|
||||
* Translate provider-neutral history into the Swift request: everything but
|
||||
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||
* the model answers.
|
||||
*/
|
||||
export function buildSystemModelRequest(
|
||||
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||
): SystemModelRequest {
|
||||
const messages = options.messages;
|
||||
const last = messages[messages.length - 1];
|
||||
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||
const history: HistoryEntry[] = [];
|
||||
const toolNames = new Map<string, string>();
|
||||
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||
appendHistory(history, message, toolNames);
|
||||
}
|
||||
const tools = (options.tools ?? []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
}));
|
||||
return {
|
||||
system: options.system?.trim() || "You are a helpful assistant.",
|
||||
tools,
|
||||
history,
|
||||
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||
permissive: tools.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||
if (message.role === "user") {
|
||||
for (const result of toolResultsOf(message)) {
|
||||
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "prompt", text });
|
||||
return;
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "response", text });
|
||||
if (typeof message.content === "string") return;
|
||||
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||
for (const block of message.content) {
|
||||
if (block.type !== "tool_use") continue;
|
||||
toolNames.set(block.id, block.name);
|
||||
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||
}
|
||||
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||
}
|
||||
|
||||
function textOf(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content.trim();
|
||||
return content
|
||||
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||
}
|
||||
|
||||
function parseArguments(json: string): any {
|
||||
try {
|
||||
const value = JSON.parse(json);
|
||||
return value && typeof value === "object" ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
type NativeEvent = {
|
||||
kind: string;
|
||||
text?: string;
|
||||
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Generations parked in tool calls, by call id. */
|
||||
const parked = new Map<string, Generation>();
|
||||
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||
const live = new Set<Generation>();
|
||||
|
||||
/**
|
||||
* One native generation, possibly spanning several loop iterations: each
|
||||
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||
* next one resumes it with the results.
|
||||
*/
|
||||
class Generation {
|
||||
private readonly native: any;
|
||||
private options: LlmStreamOptions | null;
|
||||
private iteration = 0;
|
||||
private text = "";
|
||||
private callIds: string[] = [];
|
||||
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||
private closed = false;
|
||||
|
||||
constructor(options: LlmStreamOptions) {
|
||||
this.options = options;
|
||||
this.native = FaceclawSystemLanguageModel.new();
|
||||
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||
live.add(this);
|
||||
}
|
||||
|
||||
start(request: SystemModelRequest): LlmStreamHandle {
|
||||
this.native.start(JSON.stringify(request));
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
/** Whether these results answer this generation's parked calls. */
|
||||
awaits(results: ToolResultBlock[]): boolean {
|
||||
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||
}
|
||||
|
||||
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||
this.unpark();
|
||||
this.options = options;
|
||||
this.iteration++;
|
||||
this.text = "";
|
||||
this.native.resolveToolCalls(
|
||||
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||
);
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
try {
|
||||
this.native.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private handle(): LlmStreamHandle {
|
||||
const iteration = this.iteration;
|
||||
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||
// since then: its owner gave up on the turn.
|
||||
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||
}
|
||||
|
||||
private close(): void {
|
||||
this.closed = true;
|
||||
this.unpark();
|
||||
this.options = null;
|
||||
this.native.eventHandler = null;
|
||||
live.delete(this);
|
||||
}
|
||||
|
||||
private park(ids: string[]): void {
|
||||
this.callIds = ids;
|
||||
for (const id of ids) parked.set(id, this);
|
||||
this.expiry = setTimeout(() => {
|
||||
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||
this.cancel();
|
||||
}, PARKED_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private unpark(): void {
|
||||
for (const id of this.callIds) {
|
||||
if (parked.get(id) === this) parked.delete(id);
|
||||
}
|
||||
this.callIds = [];
|
||||
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||
this.expiry = null;
|
||||
}
|
||||
|
||||
private onEvent(json: string): void {
|
||||
if (this.closed) return;
|
||||
let event: NativeEvent;
|
||||
try {
|
||||
event = JSON.parse(json);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "log") {
|
||||
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||
return;
|
||||
}
|
||||
const options = this.options;
|
||||
if (!options) return;
|
||||
switch (event.kind) {
|
||||
case "delta": {
|
||||
const delta = event.text ?? "";
|
||||
this.text += delta;
|
||||
options.onTextDelta?.(delta, this.text);
|
||||
return;
|
||||
}
|
||||
case "toolCalls": {
|
||||
const calls = event.calls ?? [];
|
||||
const text = this.text.trim();
|
||||
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||
for (const call of calls) {
|
||||
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||
}
|
||||
this.options = null;
|
||||
this.park(calls.map((call) => call.id));
|
||||
options.onDone({ text, content, stopReason: "tool_use" });
|
||||
return;
|
||||
}
|
||||
case "done": {
|
||||
this.close();
|
||||
const text = this.text.trim();
|
||||
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||
return;
|
||||
}
|
||||
case "error": {
|
||||
this.close();
|
||||
if (event.code === "refusal") {
|
||||
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||
} else {
|
||||
options.onError(event.message || "The on-device model failed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a reply from the system model. A request whose history ends in tool
|
||||
* results for a parked generation resumes it; anything else starts afresh.
|
||||
*/
|
||||
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||
const availability = localModelAvailability();
|
||||
if (availability !== "available") {
|
||||
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||
let generation: Generation | null = null;
|
||||
try {
|
||||
generation = new Generation(options);
|
||||
return generation.start(buildSystemModelRequest(options));
|
||||
} catch (error) {
|
||||
generation?.cancel();
|
||||
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The on-phone LLM: one entry point for every feature that can run on the
|
||||
* phone without an API key (the direct assistant's "local" provider,
|
||||
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||
* contract, plus model status for Settings.
|
||||
*
|
||||
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||
*/
|
||||
export {
|
||||
LOCAL_MODEL,
|
||||
isLocalModelReady,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
unloadLocalModel,
|
||||
streamLocalQwen as streamLocalModel,
|
||||
} from "./llama";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -6,6 +6,9 @@ import {
|
||||
type MediaBrowserApp,
|
||||
type MediaBrowserBridge,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify.ios";
|
||||
import type { SpotifyLibrary } from "./spotify-library";
|
||||
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
|
||||
|
||||
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
|
||||
|
||||
@@ -30,7 +33,7 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS"));
|
||||
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Not an Apple Music library"));
|
||||
if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed"));
|
||||
this.connected = true;
|
||||
return Promise.resolve("root");
|
||||
@@ -67,4 +70,42 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge();
|
||||
/** Apple Music's library, plus the Spotify library once signed in. */
|
||||
export class IosMediaBrowserBridge implements MediaBrowserBridge {
|
||||
private current: MediaBrowserBridge | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly appleMusic: AppleMusicBrowserBridge,
|
||||
private readonly spotify: () => SpotifyLibrary | null,
|
||||
) {}
|
||||
|
||||
listBrowsableApps(): MediaBrowserApp[] {
|
||||
return [...this.appleMusic.listBrowsableApps(), ...(this.spotify()?.listBrowsableApps() ?? [])];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
const library = app.packageName === SPOTIFY_IOS_BUNDLE_ID ? this.spotify() : this.appleMusic;
|
||||
if (!library) return Promise.reject(new Error(`${app.appName} can't be browsed`));
|
||||
this.current = library;
|
||||
return library.connect(app);
|
||||
}
|
||||
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
this.current?.playFromMediaId(mediaId);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.current?.disconnect();
|
||||
this.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new IosMediaBrowserBridge(
|
||||
new AppleMusicBrowserBridge(),
|
||||
() => spotifyService()?.library ?? null,
|
||||
);
|
||||
@@ -2,6 +2,9 @@ import { Utils } from "@nativescript/core";
|
||||
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types";
|
||||
import { spotifyService } from "./spotify";
|
||||
import type { SpotifyLibrary } from "./spotify-library";
|
||||
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -157,4 +160,54 @@ export class FaceclawMediaBrowserBridge implements MediaBrowserBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge();
|
||||
/**
|
||||
* The Android libraries: apps' media browser services, with Spotify's
|
||||
* library from its Web API instead. Spotify's own browser service admits
|
||||
* only allowlisted callers (Android Auto, Wear, Assistant), so connecting to
|
||||
* it always fails; signed in, its row opens the Web API library, and signed
|
||||
* out, opening it says how to sign in. Signed in with Spotify not installed,
|
||||
* the library is still offered (it plays on the user's other devices).
|
||||
*/
|
||||
export class AndroidMediaBrowserBridge implements MediaBrowserBridge {
|
||||
private current: MediaBrowserBridge | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly services: MediaBrowserBridge,
|
||||
private readonly spotify: () => SpotifyLibrary | null,
|
||||
private readonly isEnabled: (packageName: string) => boolean = isMediaAppEnabled,
|
||||
) {}
|
||||
|
||||
listBrowsableApps(refresh = false): MediaBrowserApp[] {
|
||||
const apps = this.services.listBrowsableApps(refresh);
|
||||
const library = this.spotify();
|
||||
if (!library || apps.some((app) => app.packageName === SPOTIFY_ANDROID_PACKAGE) || !this.isEnabled(SPOTIFY_ANDROID_PACKAGE)) {
|
||||
return apps;
|
||||
}
|
||||
return [...apps, ...library.listBrowsableApps()];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
const library = app.packageName === SPOTIFY_ANDROID_PACKAGE ? this.spotify() : null;
|
||||
this.current = library ?? this.services;
|
||||
return this.current.connect(library ? library.app : app);
|
||||
}
|
||||
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
this.current?.playFromMediaId(mediaId);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.current?.disconnect();
|
||||
this.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new AndroidMediaBrowserBridge(
|
||||
new FaceclawMediaBrowserBridge(),
|
||||
() => spotifyService()?.library ?? null,
|
||||
);
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify.ios";
|
||||
import type { SpotifyPlayer } from "./spotify-player";
|
||||
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
@@ -88,7 +90,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
void this.start();
|
||||
this.stateListeners.add(listener);
|
||||
this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS);
|
||||
this.pollTimer ??= setInterval(() => this.publish(), POLL_MS);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.stateListeners.delete(listener);
|
||||
@@ -142,7 +144,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
if (this.hasAccess()) return Promise.resolve(true);
|
||||
this.pendingAccess ??= this.askForAccess().finally(() => {
|
||||
this.pendingAccess = null;
|
||||
this.refresh();
|
||||
this.publish();
|
||||
});
|
||||
return this.pendingAccess;
|
||||
}
|
||||
@@ -187,12 +189,12 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
const music = FaceclawAppleMusic.shared();
|
||||
if (!this.bound) {
|
||||
this.bound = true;
|
||||
music.changeHandler = () => this.refresh();
|
||||
music.changeHandler = () => this.publish();
|
||||
}
|
||||
return music;
|
||||
}
|
||||
|
||||
private refresh(): void {
|
||||
private publish(): void {
|
||||
if (!this.stateListeners.size) return;
|
||||
const state = this.snapshot();
|
||||
const signature = stateSignature(state);
|
||||
@@ -228,4 +230,192 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge();
|
||||
export type IosMediaSource = "apple-music" | "spotify";
|
||||
|
||||
/**
|
||||
* Which player to follow: the one playing; with both playing, the current
|
||||
* choice (no flapping); with neither, the one that played last, and before
|
||||
* either has played this run, Spotify when it reports a session (its player
|
||||
* reports only a recently active one, while the system Music player keeps
|
||||
* its last item for days).
|
||||
*/
|
||||
export function chooseMediaSource(
|
||||
current: IosMediaSource,
|
||||
appleMusic: MediaControllerState,
|
||||
spotify: MediaControllerState | null,
|
||||
appleMusicLastPlayingAtMs: number,
|
||||
spotifyLastPlayingAtMs: number,
|
||||
): IosMediaSource {
|
||||
if (!spotify?.accessEnabled) return "apple-music";
|
||||
if (!appleMusic.accessEnabled) return "spotify";
|
||||
const applePlaying = appleMusic.playbackState === "playing";
|
||||
const spotifyPlaying = spotify.playbackState === "playing";
|
||||
if (applePlaying && spotifyPlaying) return current;
|
||||
if (applePlaying) return "apple-music";
|
||||
if (spotifyPlaying) return "spotify";
|
||||
if (appleMusicLastPlayingAtMs !== spotifyLastPlayingAtMs) {
|
||||
return spotifyLastPlayingAtMs > appleMusicLastPlayingAtMs ? "spotify" : "apple-music";
|
||||
}
|
||||
return spotify.available ? "spotify" : "apple-music";
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS Music bridge: Apple Music (the system player), plus Spotify once
|
||||
* signed in (its Web API: playback on this phone or any Spotify Connect
|
||||
* device). The state and every command follow one of them, by
|
||||
* chooseMediaSource.
|
||||
*/
|
||||
export class IosMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt =
|
||||
"Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone. For Spotify, sign in from the app menu instead.";
|
||||
readonly accessLabel: string;
|
||||
readonly accessAction: string;
|
||||
private source: IosMediaSource = "apple-music";
|
||||
private appleMusicLastPlayingAtMs = 0;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private unsubscribeSources: Array<() => void> = [];
|
||||
/** The source and its artVersion the composite artVersion was bumped for. */
|
||||
private artKey = "";
|
||||
private artVersion = 0;
|
||||
|
||||
constructor(
|
||||
private readonly appleMusic: AppleMusicControllerBridge,
|
||||
private readonly spotify: () => SpotifyPlayer | null,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {
|
||||
this.accessLabel = appleMusic.accessLabel;
|
||||
this.accessAction = appleMusic.accessAction;
|
||||
}
|
||||
|
||||
get skipRebuildsQueue(): boolean {
|
||||
return this.active().skipRebuildsQueue;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
if (!this.listeners.size) {
|
||||
const update = () => this.emit();
|
||||
const spotify = this.spotify();
|
||||
this.unsubscribeSources = [this.appleMusic.onStateChange(update), ...(spotify ? [spotify.onStateChange(update)] : [])];
|
||||
}
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (this.listeners.size) return;
|
||||
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
|
||||
this.unsubscribeSources = [];
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const appleMusic = this.appleMusic.snapshot();
|
||||
const spotify = this.spotify()?.snapshot() ?? null;
|
||||
this.select(appleMusic, spotify);
|
||||
const state = this.source === "spotify" && spotify ? spotify : appleMusic;
|
||||
// Changes with the source as well as with that source's art.
|
||||
const artKey = `${this.source}:${state.artVersion}`;
|
||||
if (artKey !== this.artKey) {
|
||||
this.artKey = artKey;
|
||||
this.artVersion++;
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
status: state.available ? state.status : idleStatus(appleMusic, spotify, state),
|
||||
artVersion: this.artVersion,
|
||||
// A Spotify command can fail while Apple Music is the one shown (playing from the Spotify library).
|
||||
notice: spotify?.notice || state.notice || "",
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.appleMusic.start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.appleMusic.stop();
|
||||
}
|
||||
|
||||
playPause(): Promise<void> {
|
||||
return this.active().playPause();
|
||||
}
|
||||
|
||||
skipNext(): Promise<void> {
|
||||
return this.active().skipNext();
|
||||
}
|
||||
|
||||
skipPrevious(): Promise<void> {
|
||||
return this.active().skipPrevious();
|
||||
}
|
||||
|
||||
/** Apple Music's media-library permission (what requestAccess asks for); Spotify signs in from the Music menu. */
|
||||
hasAccess(): boolean {
|
||||
return this.appleMusic.hasAccess();
|
||||
}
|
||||
|
||||
requestAccess(): Promise<boolean> {
|
||||
return this.appleMusic.requestAccess();
|
||||
}
|
||||
|
||||
skipToQueueItem(id: number): Promise<void> {
|
||||
return this.active().skipToQueueItem(id);
|
||||
}
|
||||
|
||||
getMediaVolumePercent(): number {
|
||||
return this.active().getMediaVolumePercent();
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
this.active().setMediaVolumePercent(volumePercent);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
return this.active().getAlbumArt(maxSize);
|
||||
}
|
||||
|
||||
getQueue(): MediaQueueItem[] {
|
||||
return this.active().getQueue();
|
||||
}
|
||||
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
await this.active().sendCustomAction?.(action);
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.spotify()?.refresh();
|
||||
}
|
||||
|
||||
private active(): MediaControllerBridge {
|
||||
const spotify = this.spotify();
|
||||
this.select(this.appleMusic.snapshot(), spotify?.snapshot() ?? null);
|
||||
return this.source === "spotify" && spotify ? spotify : this.appleMusic;
|
||||
}
|
||||
|
||||
private select(appleMusic: MediaControllerState, spotify: MediaControllerState | null): void {
|
||||
if (appleMusic.playbackState === "playing") this.appleMusicLastPlayingAtMs = this.now();
|
||||
this.source = chooseMediaSource(this.source, appleMusic, spotify, this.appleMusicLastPlayingAtMs,
|
||||
this.spotify()?.lastPlayingAtMs ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward every source update: each source already drops repeats, and
|
||||
* Spotify also signals changes that are not in the state (its queue).
|
||||
*/
|
||||
private emit(): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/** The no-session line: which players were checked. */
|
||||
function idleStatus(appleMusic: MediaControllerState, spotify: MediaControllerState | null, shown: MediaControllerState): string {
|
||||
if (appleMusic.accessEnabled && spotify?.accessEnabled) return "Nothing is playing.";
|
||||
return shown.status;
|
||||
}
|
||||
|
||||
export const appleMusicControllerBridge = new AppleMusicControllerBridge();
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new IosMediaControllerBridge(
|
||||
appleMusicControllerBridge,
|
||||
() => spotifyService()?.player ?? null,
|
||||
);
|
||||
@@ -12,11 +12,14 @@ import {
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify";
|
||||
import { SPOTIFY_ANDROID_PACKAGE, type SpotifyPlayer } from "./spotify-player";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
/** Android media sessions (any player that publishes one), through FaceclawMediaController. */
|
||||
export class FaceclawMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings.";
|
||||
readonly accessLabel = "Notification access needed";
|
||||
@@ -215,4 +218,251 @@ function normalizePlaybackState(value: string): MediaPlaybackState {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge();
|
||||
export type AndroidMediaSource = "session" | "spotify";
|
||||
|
||||
/**
|
||||
* Which to follow: the media session (any player, Spotify included when it
|
||||
* runs on this phone) or Spotify's Web API player (signed in; it also sees
|
||||
* Spotify playing on a computer or speaker). The one playing; with both
|
||||
* playing, the session when it is Spotify's own (the same playback), else the
|
||||
* current choice (no flapping); with neither, a Spotify session, then the one
|
||||
* that played last, then whichever has something. Without notification
|
||||
* access the session shows its access prompt unless Spotify has something.
|
||||
*/
|
||||
export function chooseAndroidMediaSource(
|
||||
current: AndroidMediaSource,
|
||||
session: MediaControllerState,
|
||||
spotify: MediaControllerState | null,
|
||||
sessionLastPlayingAtMs: number,
|
||||
spotifyLastPlayingAtMs: number,
|
||||
): AndroidMediaSource {
|
||||
if (!spotify?.accessEnabled) return "session";
|
||||
if (!session.accessEnabled) return spotify.available ? "spotify" : "session";
|
||||
const sessionIsSpotify = session.packageName === SPOTIFY_ANDROID_PACKAGE;
|
||||
const sessionPlaying = session.playbackState === "playing";
|
||||
const spotifyPlaying = spotify.playbackState === "playing";
|
||||
if (sessionPlaying && spotifyPlaying) return sessionIsSpotify ? "session" : current;
|
||||
if (sessionPlaying) return "session";
|
||||
if (spotifyPlaying) return "spotify";
|
||||
if (sessionIsSpotify && session.available) return "session";
|
||||
if (sessionLastPlayingAtMs !== spotifyLastPlayingAtMs) {
|
||||
return spotifyLastPlayingAtMs > sessionLastPlayingAtMs ? "spotify" : "session";
|
||||
}
|
||||
return session.available || !spotify.available ? "session" : "spotify";
|
||||
}
|
||||
|
||||
/**
|
||||
* The Android Music bridge: media sessions, plus Spotify's Web API once
|
||||
* signed in (optional). Spotify playing on this phone stays the session's:
|
||||
* its state, art, queue and transport come from the session (instant, no
|
||||
* quota), and the Web API adds what the session lacks: shuffle / repeat /
|
||||
* like rows and the volume of a Spotify Connect speaker or computer. Spotify
|
||||
* playing on another device, with no session here, follows the Web API player.
|
||||
*/
|
||||
export class AndroidMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt: string;
|
||||
readonly accessLabel: string;
|
||||
readonly accessAction: string;
|
||||
private source: AndroidMediaSource = "session";
|
||||
private sessionLastPlayingAtMs = 0;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private unsubscribeSources: Array<() => void> = [];
|
||||
/** The source and its artVersion the composite artVersion was bumped for. */
|
||||
private artKey = "";
|
||||
private artVersion = 0;
|
||||
/** The player the last getQueue listed, so a jump goes to the same one. */
|
||||
private queueSource: MediaControllerBridge | null = null;
|
||||
/** A Spotify session's item and play state, to re-read the Web API when they change. */
|
||||
private spotifySessionKey = "";
|
||||
|
||||
constructor(
|
||||
private readonly session: MediaControllerBridge,
|
||||
private readonly spotify: () => SpotifyPlayer | null,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {
|
||||
this.accessPrompt = session.accessPrompt;
|
||||
this.accessLabel = session.accessLabel;
|
||||
this.accessAction = session.accessAction;
|
||||
}
|
||||
|
||||
get skipRebuildsQueue(): boolean {
|
||||
return (this.queueSource ?? this.active()).skipRebuildsQueue;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
if (!this.listeners.size) {
|
||||
const spotify = this.spotify();
|
||||
this.unsubscribeSources = [
|
||||
this.session.onStateChange((state) => this.sessionChanged(state)),
|
||||
...(spotify ? [spotify.onStateChange(() => this.emit())] : []),
|
||||
];
|
||||
}
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (this.listeners.size) return;
|
||||
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
|
||||
this.unsubscribeSources = [];
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const session = this.session.snapshot();
|
||||
const spotify = this.spotify()?.snapshot() ?? null;
|
||||
this.select(session, spotify);
|
||||
const state = this.source === "spotify" && spotify ? spotify : withSpotifyExtras(session, spotify);
|
||||
// Changes with the source as well as with that source's art.
|
||||
const artKey = `${this.source}:${state.artVersion}`;
|
||||
if (artKey !== this.artKey) {
|
||||
this.artKey = artKey;
|
||||
this.artVersion++;
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
artVersion: this.artVersion,
|
||||
// A Spotify command can fail while the session is the one shown (playing from the Spotify library).
|
||||
notice: spotify?.notice || state.notice || "",
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.session.start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.session.stop();
|
||||
}
|
||||
|
||||
playPause(): Promise<void> {
|
||||
return this.active().playPause();
|
||||
}
|
||||
|
||||
skipNext(): Promise<void> {
|
||||
return this.active().skipNext();
|
||||
}
|
||||
|
||||
skipPrevious(): Promise<void> {
|
||||
return this.active().skipPrevious();
|
||||
}
|
||||
|
||||
/** Notification access (what requestAccess opens); Spotify signs in from the Music menu. */
|
||||
hasAccess(): boolean {
|
||||
return this.session.hasAccess();
|
||||
}
|
||||
|
||||
requestAccess(): Promise<boolean> {
|
||||
return this.session.requestAccess();
|
||||
}
|
||||
|
||||
skipToQueueItem(id: number): Promise<void> {
|
||||
return (this.queueSource ?? this.active()).skipToQueueItem(id);
|
||||
}
|
||||
|
||||
/** Spotify's own volume logic (a Connect device's volume, else this phone's) whenever Spotify is shown. */
|
||||
getMediaVolumePercent(): number {
|
||||
return this.volumeSource().getMediaVolumePercent();
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
this.volumeSource().setMediaVolumePercent(volumePercent);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
return this.active().getAlbumArt(maxSize);
|
||||
}
|
||||
|
||||
/** The session's queue; for a Spotify session that lists none, the Web API's. */
|
||||
getQueue(): MediaQueueItem[] {
|
||||
const active = this.active();
|
||||
const queue = active.getQueue();
|
||||
const spotify = this.spotifyExtras();
|
||||
if (queue.length || active !== this.session || !spotify) {
|
||||
this.queueSource = active;
|
||||
return queue;
|
||||
}
|
||||
this.queueSource = spotify;
|
||||
return spotify.getQueue();
|
||||
}
|
||||
|
||||
/** Spotify's actions (shown with a Spotify session too) go to the Web API player; any others to the session. */
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
const spotify = this.spotify();
|
||||
if (spotify && spotify.snapshot().customActions?.some((custom) => custom.action === action)) {
|
||||
await spotify.sendCustomAction(action);
|
||||
} else {
|
||||
await this.session.sendCustomAction?.(action);
|
||||
}
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.spotify()?.refresh();
|
||||
}
|
||||
|
||||
private active(): MediaControllerBridge {
|
||||
const spotify = this.spotify();
|
||||
this.select(this.session.snapshot(), spotify?.snapshot() ?? null);
|
||||
return this.source === "spotify" && spotify ? spotify : this.session;
|
||||
}
|
||||
|
||||
/** The signed-in Web API player when it describes what the session shows (Spotify's own session); else null. */
|
||||
private spotifyExtras(): SpotifyPlayer | null {
|
||||
const spotify = this.spotify();
|
||||
if (!spotify || this.session.snapshot().packageName !== SPOTIFY_ANDROID_PACKAGE) return null;
|
||||
return spotify.snapshot().available ? spotify : null;
|
||||
}
|
||||
|
||||
private volumeSource(): MediaControllerBridge {
|
||||
const active = this.active();
|
||||
return active === this.session ? this.spotifyExtras() ?? this.session : active;
|
||||
}
|
||||
|
||||
private select(session: MediaControllerState, spotify: MediaControllerState | null): void {
|
||||
if (session.playbackState === "playing") this.sessionLastPlayingAtMs = this.now();
|
||||
this.source = chooseAndroidMediaSource(this.source, session, spotify, this.sessionLastPlayingAtMs,
|
||||
this.spotify()?.lastPlayingAtMs ?? 0);
|
||||
}
|
||||
|
||||
private sessionChanged(state: MediaControllerState): void {
|
||||
// Spotify's session changed track or started / stopped: re-read the Web
|
||||
// API now, so its rows (like this song, shuffle) follow at once rather
|
||||
// than at the next poll. Signed out, the player skips the request.
|
||||
const key = state.packageName === SPOTIFY_ANDROID_PACKAGE ? `${state.title}\n${state.playbackState}` : "";
|
||||
if (key !== this.spotifySessionKey) {
|
||||
this.spotifySessionKey = key;
|
||||
if (key) void this.spotify()?.refresh();
|
||||
}
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/** Forward every source update: each drops its own repeats, and Spotify also signals queue changes. */
|
||||
private emit(): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spotify's session state plus what only the Web API knows: its custom rows
|
||||
* and, when it plays elsewhere, the device ("Spotify · Kitchen"). "Like
|
||||
* song" names the Web API's current item, so it is left out until both agree
|
||||
* on the track (the API lags a skip by up to a poll).
|
||||
*/
|
||||
export function withSpotifyExtras(session: MediaControllerState, spotify: MediaControllerState | null): MediaControllerState {
|
||||
if (!spotify?.available || session.packageName !== SPOTIFY_ANDROID_PACKAGE) return session;
|
||||
const sameItem = spotify.title === session.title;
|
||||
const actions = (spotify.customActions ?? []).filter((custom) => sameItem || custom.action !== "like");
|
||||
return {
|
||||
...session,
|
||||
appName: sameItem ? spotify.appName : session.appName,
|
||||
customActions: [...(session.customActions ?? []), ...actions],
|
||||
};
|
||||
}
|
||||
|
||||
const sessionBridge = new FaceclawMediaControllerBridge();
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new AndroidMediaControllerBridge(
|
||||
sessionBridge,
|
||||
() => spotifyService()?.player ?? null,
|
||||
);
|
||||
@@ -3,7 +3,8 @@ import type { GrayImage } from "../graphics/image";
|
||||
/**
|
||||
* Shapes shared by the media bridges: media-controller.ts / media-browser.ts
|
||||
* drive Android media sessions and MediaBrowserServices; their .ios.ts
|
||||
* counterparts drive Apple Music through the MediaPlayer framework.
|
||||
* counterparts drive Apple Music through the MediaPlayer framework. Both
|
||||
* add Spotify through its Web API once signed in (spotify-*.ts).
|
||||
*/
|
||||
|
||||
/** The player behind the iOS bridges: the system Music app. */
|
||||
@@ -13,9 +14,19 @@ export const APPLE_MUSIC_PACKAGE = "com.apple.Music";
|
||||
export type MediaQueueItem = {
|
||||
id: number;
|
||||
title: string;
|
||||
/** Usually the artist; absent when the player gives none. */
|
||||
subtitle?: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
/** A player-specific action beyond the transport controls (Spotify's shuffle, repeat, like). */
|
||||
export type MediaCustomAction = {
|
||||
/** Stable id to send back with sendCustomAction. */
|
||||
action: string;
|
||||
/** The row label, naming the current state where there is one ("Shuffle: on"). */
|
||||
name: string;
|
||||
};
|
||||
|
||||
export type MediaPlaybackState =
|
||||
| "notification-access-required"
|
||||
| "idle"
|
||||
@@ -47,6 +58,10 @@ export type MediaControllerState = {
|
||||
* players re-publish the same strings with a different bitmap.
|
||||
*/
|
||||
artVersion: number;
|
||||
/** Player-specific actions (see sendCustomAction); absent when the player offers none. */
|
||||
customActions?: MediaCustomAction[];
|
||||
/** A short-lived message about a command that failed (Spotify needs Premium, ...); absent or "" otherwise. */
|
||||
notice?: string;
|
||||
};
|
||||
|
||||
export const DEFAULT_MEDIA_STATE: MediaControllerState = {
|
||||
@@ -113,6 +128,13 @@ export interface MediaControllerBridge {
|
||||
getAlbumArt(maxSize: number): GrayImage | null;
|
||||
/** The player's queue (playlist), empty when the player exposes none. */
|
||||
getQueue(): MediaQueueItem[];
|
||||
/** Run one of the state's customActions. */
|
||||
sendCustomAction?(action: string): Promise<void>;
|
||||
/**
|
||||
* Re-read the player now, for one-off readers (the assistant) when no
|
||||
* listener keeps the state current; absent when snapshots are always live.
|
||||
*/
|
||||
refresh?(): Promise<void>;
|
||||
}
|
||||
|
||||
export type MediaBrowserApp = {
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/** The iOS twin of mic-control-link.ts, over the running shared-Kotlin session. */
|
||||
|
||||
export function micControlSession(): unknown {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendMicControlRecord(
|
||||
session: unknown,
|
||||
record: Uint8Array,
|
||||
label: string,
|
||||
rightTemple: boolean,
|
||||
leftTemple: boolean,
|
||||
): boolean {
|
||||
return (session as FaceclawCommunicatorBridge).sendFaceclawMicControl(record, label, rightTemple, leftTemple);
|
||||
}
|
||||
|
||||
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
|
||||
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const unsubscribe = (session as FaceclawCommunicatorBridge).addMicStatusListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { toJavaBytes } from "./cloud-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Platform plumbing for the CFW mic_control channel (apps/microphones/mic-control.ts):
|
||||
* the running session to send records on and to hear field-104 status replies
|
||||
* from. The session is opaque and is a different object after a reconnect.
|
||||
*/
|
||||
|
||||
export function micControlSession(): unknown {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendMicControlRecord(
|
||||
session: unknown,
|
||||
record: Uint8Array,
|
||||
label: string,
|
||||
rightTemple: boolean,
|
||||
leftTemple: boolean,
|
||||
): boolean {
|
||||
return (session as any).sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
}
|
||||
|
||||
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
|
||||
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const communicator = session as any;
|
||||
const proxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
||||
onMicStatus: (body: any, arm: string) => listener(toUint8Array(body), String(arm)),
|
||||
});
|
||||
communicator.addMicStatusListener(proxy);
|
||||
return () => {
|
||||
try {
|
||||
communicator.removeMicStatusListener(proxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import type { PebbleIconName } from "../graphics/pebble-icons";
|
||||
|
||||
/**
|
||||
* Icons for notification sources that arrive without one: iPhone
|
||||
* notifications over ANCS carry the app's bundle ID, its display name
|
||||
* (fetched separately) and an ANCS category, but no icon. The icons are
|
||||
* Pebble's (see scripts/generate-pebble-icons.cjs).
|
||||
*
|
||||
* The bundle IDs start from the Pebble firmware's own ANCS table (PebbleOS
|
||||
* include/pbl/services/notifications/ancs/ancs_known_apps.h, minus retired
|
||||
* apps), plus Apple's built-in apps and popular apps mapped to a generic
|
||||
* pictogram. Every App Store bundle ID here was checked against Apple's
|
||||
* iTunes lookup when it was added; Apple's built-in apps follow
|
||||
* github.com/joeblau/apple-bundle-identifiers.
|
||||
*
|
||||
* Display names are a fallback for apps whose bundle ID varies per build:
|
||||
* self-built apps such as Loop sign with their builder's team ID. They match
|
||||
* the whole name, ignoring case, so generic words (an app called "United"
|
||||
* or "Voice") are left out. Bundle prefixes catch regional variants
|
||||
* (com.abbott.librelink.us, .gb, ...).
|
||||
*
|
||||
* Apple's Phone app is deliberately absent: its notifications are calls,
|
||||
* missed calls and voicemail, which the ANCS category tells apart.
|
||||
*/
|
||||
type AppIcon = {
|
||||
icon: PebbleIconName;
|
||||
bundleIds: readonly string[];
|
||||
bundlePrefixes?: readonly string[];
|
||||
names?: readonly string[];
|
||||
};
|
||||
|
||||
const APP_ICONS: readonly AppIcon[] = [
|
||||
// Apple's built-in apps.
|
||||
{ icon: "sms", bundleIds: ["com.apple.MobileSMS"] },
|
||||
{ icon: "email", bundleIds: ["com.apple.mobilemail"] },
|
||||
{ icon: "calendar", bundleIds: ["com.apple.mobilecal"] },
|
||||
{ icon: "reminder", bundleIds: ["com.apple.reminders"] },
|
||||
{ icon: "facetime", bundleIds: ["com.apple.facetime"] },
|
||||
{ icon: "iosPhotos", bundleIds: ["com.apple.mobileslideshow"] },
|
||||
{ icon: "alarmClock", bundleIds: ["com.apple.mobiletimer"] },
|
||||
{ icon: "weather", bundleIds: ["com.apple.weather"] },
|
||||
{ icon: "location", bundleIds: ["com.apple.findmy", "com.apple.Maps"] },
|
||||
{ icon: "settings", bundleIds: ["com.apple.Preferences"] },
|
||||
{ icon: "heart", bundleIds: ["com.apple.Health"] },
|
||||
{ icon: "activity", bundleIds: ["com.apple.Fitness"] },
|
||||
{ icon: "news", bundleIds: ["com.apple.news"] },
|
||||
{ icon: "podcast", bundleIds: ["com.apple.podcasts"] },
|
||||
{ icon: "music", bundleIds: ["com.apple.Music"] },
|
||||
{ icon: "tv", bundleIds: ["com.apple.tv"] },
|
||||
{ icon: "payBill", bundleIds: ["com.apple.Passbook"] },
|
||||
{ icon: "stocks", bundleIds: ["com.apple.stocks"] },
|
||||
{ icon: "appStore", bundleIds: ["com.apple.AppStore"] },
|
||||
{ icon: "appleShortcuts", bundleIds: ["com.apple.shortcuts"] },
|
||||
{ icon: "microphone", bundleIds: ["com.apple.VoiceMemos"] },
|
||||
{ icon: "contacts", bundleIds: ["com.apple.MobileAddressBook"] },
|
||||
{ icon: "shopping", bundleIds: ["com.apple.store.Jolly"] },
|
||||
|
||||
// Apps with an icon of their own.
|
||||
{ icon: "airmail", bundleIds: ["com.airmailapp.iphone"], names: ["Airmail"] },
|
||||
{ icon: "amazon", bundleIds: ["com.amazon.Amazon"], names: ["Amazon"] },
|
||||
{ icon: "bbcNews", bundleIds: ["uk.co.bbc.news"], names: ["BBC News"] },
|
||||
{ icon: "beeper", bundleIds: ["com.automattic.beeper"], names: ["Beeper"] },
|
||||
{ icon: "bluesky", bundleIds: ["xyz.blueskyweb.app"], names: ["Bluesky"] },
|
||||
{ icon: "discord", bundleIds: ["com.hammerandchisel.discord"], names: ["Discord"] },
|
||||
{ icon: "duolingo", bundleIds: ["com.duolingo.DuolingoMobile"], names: ["Duolingo"] },
|
||||
{ icon: "ebay", bundleIds: ["com.ebay.iphone"], names: ["eBay"] },
|
||||
{ icon: "element", bundleIds: ["im.vector.app", "io.element.elementx"], names: ["Element", "Element X", "Element Classic"] },
|
||||
{ icon: "facebook", bundleIds: ["com.facebook.Facebook"], names: ["Facebook"] },
|
||||
{ icon: "gmail", bundleIds: ["com.google.Gmail"], names: ["Gmail"] },
|
||||
{ icon: "googleChat", bundleIds: ["com.google.Dynamite"], names: ["Google Chat"] },
|
||||
{ icon: "googleMaps", bundleIds: ["com.google.Maps"], names: ["Google Maps"] },
|
||||
{ icon: "googlePhotos", bundleIds: ["com.google.photos"], names: ["Google Photos"] },
|
||||
{ icon: "googleTasks", bundleIds: ["com.google.tasks"], names: ["Google Tasks"] },
|
||||
{ icon: "homeAssistant", bundleIds: ["io.robbie.HomeAssistant"], names: ["Home Assistant"] },
|
||||
{ icon: "instagram", bundleIds: ["com.burbn.instagram"], names: ["Instagram"] },
|
||||
{ icon: "kakaoTalk", bundleIds: ["com.iwilab.KakaoTalk"], names: ["KakaoTalk"] },
|
||||
{ icon: "kik", bundleIds: ["com.kik.chat"], names: ["Kik"] },
|
||||
{ icon: "line", bundleIds: ["jp.naver.line"], names: ["LINE"] },
|
||||
{ icon: "linkedIn", bundleIds: ["com.linkedin.LinkedIn"], names: ["LinkedIn"] },
|
||||
{ icon: "messenger", bundleIds: ["com.facebook.Messenger"], names: ["Messenger"] },
|
||||
{ icon: "omniFocus", bundleIds: ["com.omnigroup.OmniFocus4", "com.omnigroup.OmniFocus3.iOS"], names: ["OmniFocus"] },
|
||||
{ icon: "outlook", bundleIds: ["com.microsoft.Office.Outlook"], names: ["Outlook"] },
|
||||
{ icon: "reddit", bundleIds: ["com.reddit.Reddit"], names: ["Reddit"] },
|
||||
{ icon: "signal", bundleIds: ["org.whispersystems.signal"], names: ["Signal"] },
|
||||
{ icon: "simplex", bundleIds: ["chat.simplex.app"], names: ["SimpleX"] },
|
||||
{ icon: "slack", bundleIds: ["com.tinyspeck.chatlyio"], names: ["Slack"] },
|
||||
{ icon: "snapchat", bundleIds: ["com.toyopagroup.picaboo"], names: ["Snapchat"] },
|
||||
{ icon: "steam", bundleIds: ["com.valvesoftware.Steam"], names: ["Steam"] },
|
||||
{ icon: "substack", bundleIds: ["com.substack.Substack"], names: ["Substack"] },
|
||||
{ icon: "swarm", bundleIds: ["com.foursquare.robin"], names: ["Swarm"] },
|
||||
{ icon: "tapo", bundleIds: ["com.tplink.tapo"], names: ["Tapo"] },
|
||||
{ icon: "teams", bundleIds: ["com.microsoft.skype.teams"], names: ["Teams", "Microsoft Teams"] },
|
||||
{ icon: "telegram", bundleIds: ["ph.telegra.Telegraph"], names: ["Telegram"] },
|
||||
{ icon: "threads", bundleIds: ["com.burbn.barcelona"], names: ["Threads"] },
|
||||
{ icon: "tiktok", bundleIds: ["com.zhiliaoapp.musically"], names: ["TikTok"] },
|
||||
{ icon: "twitch", bundleIds: ["tv.twitch"], names: ["Twitch"] },
|
||||
// X, still under Twitter's bundle ID and drawn with its bird, as on Pebble.
|
||||
{ icon: "twitter", bundleIds: ["com.atebits.Tweetie2"], names: ["X", "Twitter"] },
|
||||
{ icon: "unifiProtect", bundleIds: ["com.ubnt.protect"], names: ["UniFi Protect"] },
|
||||
{ icon: "viber", bundleIds: ["com.viber"], names: ["Viber"] },
|
||||
{ icon: "waze", bundleIds: ["com.waze.iphone"], names: ["Waze"] },
|
||||
{ icon: "weChat", bundleIds: ["com.tencent.xin"], names: ["WeChat"] },
|
||||
{ icon: "whatsApp", bundleIds: ["net.whatsapp.WhatsApp"], names: ["WhatsApp"] },
|
||||
{ icon: "whatsAppBusiness", bundleIds: ["net.whatsapp.WhatsAppSMB"], names: ["WA Business", "WhatsApp Business"] },
|
||||
{ icon: "yahooMail", bundleIds: ["com.yahoo.Aerogram"], names: ["Yahoo Mail"] },
|
||||
{ icon: "youtube", bundleIds: ["com.google.ios.youtube"], names: ["YouTube"] },
|
||||
{ icon: "zoom", bundleIds: ["us.zoom.videomeetings"], names: ["Zoom"] },
|
||||
|
||||
// Apps drawn with a generic pictogram.
|
||||
{ icon: "calendar", bundleIds: ["com.google.calendar", "com.flexibits.fantastical2.iphone"], names: ["Google Calendar", "Fantastical"] },
|
||||
{ icon: "email", bundleIds: ["com.readdle.smartemail", "ch.protonmail.protonmail"], names: ["Proton Mail"] },
|
||||
{ icon: "sms", bundleIds: ["com.google.GVDialer", "ch.threema.iapp", "com.groupme.iphone-app"], names: ["Threema", "GroupMe"] },
|
||||
{ icon: "reminder", bundleIds: ["com.todoist.ios", "com.culturedcode.ThingsiPhone", "com.microsoft.to-do"], names: ["Todoist"] },
|
||||
{ icon: "car", bundleIds: ["com.ubercab.UberClient", "com.zimride.instant"], names: ["Uber", "Lyft"] },
|
||||
{ icon: "dinner", bundleIds: ["com.ubercab.UberEats", "doordash.DoorDashConsumer", "com.grubhub.search"], names: ["Uber Eats", "DoorDash", "Grubhub"] },
|
||||
{
|
||||
icon: "glucose",
|
||||
bundleIds: [
|
||||
"com.dexcom.G6", "com.dexcom.g7app", "com.dexcom.dexcomone", "com.dexcom.stelo", "com.dexcom.followglobal",
|
||||
"org.nativescript.LibreLinkUp",
|
||||
],
|
||||
bundlePrefixes: ["com.abbott.librelink.", "com.abbott.libre3."],
|
||||
names: ["Dexcom G6", "Dexcom G7", "Dexcom ONE", "LibreLink", "LibreLinkUp", "Loop", "Trio", "LoopFollow", "xDrip4iOS"],
|
||||
},
|
||||
{ icon: "music", bundleIds: ["com.spotify.client", "com.google.ios.youtubemusic", "com.pandora"], names: ["Spotify", "YouTube Music", "Pandora"] },
|
||||
{ icon: "podcast", bundleIds: ["fm.overcast.overcast", "au.com.shiftyjelly.podcasts"], names: ["Overcast", "Pocket Casts"] },
|
||||
{
|
||||
icon: "tv",
|
||||
bundleIds: ["com.netflix.Netflix", "com.hulu.plus", "com.disney.disneyplus", "com.amazon.aiv.AIVApp"],
|
||||
names: ["Netflix", "Hulu", "Disney+", "Prime Video"],
|
||||
},
|
||||
{
|
||||
icon: "payBill",
|
||||
bundleIds: [
|
||||
"net.kortina.labs.Venmo", "com.squareup.cash", "com.yourcompany.PPClient", "com.chase", "com.bankofamerica.BofA",
|
||||
"com.wf.mobilebanking", "com.capitalone.enterprisemobilebanking",
|
||||
"com.revolut.revolut", "com.transferwise.Transferwise", "de.no26.Number26", "com.bunq.ios",
|
||||
],
|
||||
names: ["Venmo", "Cash App", "PayPal", "Revolut", "N26", "bunq"],
|
||||
},
|
||||
{ icon: "stocks", bundleIds: ["com.robinhood.release.Robinhood", "com.vilcsak.bitcoin2"], names: ["Robinhood", "Coinbase"] },
|
||||
{
|
||||
icon: "flight",
|
||||
bundleIds: [
|
||||
"com.united.UnitedCustomerFacingIPhone", "com.delta.iphone.ver1", "com.aa.AmericanAirlines",
|
||||
"com.southwest.iphoneprod", "com.flightyapp.flighty",
|
||||
],
|
||||
names: ["Fly Delta", "Flighty"],
|
||||
},
|
||||
{ icon: "weather", bundleIds: ["com.weather.TWC", "com.grailr.CARROTweather"] },
|
||||
{ icon: "activity", bundleIds: ["com.strava.stravaride", "com.nike.nikeplus-gps", "com.fitbit.FitbitMobile"], names: ["Strava", "Fitbit"] },
|
||||
{ icon: "heart", bundleIds: ["com.ouraring.oura"], names: ["Oura"] },
|
||||
{ icon: "news", bundleIds: ["com.nytimes.NYTimes", "com.google.GoogleDigitalEditions"], names: ["Google News"] },
|
||||
];
|
||||
|
||||
/**
|
||||
* Icons for the ANCS categories (CategoryID: Other, IncomingCall,
|
||||
* MissedCall, Voicemail, Social, Schedule, Email, News, HealthAndFitness,
|
||||
* BusinessAndFinance, Location, Entertainment), for apps not listed above.
|
||||
*/
|
||||
const CATEGORY_ICONS: readonly PebbleIconName[] = [
|
||||
"generic", "incomingCall", "missedCall", "voicemail", "sms", "calendar",
|
||||
"email", "news", "heart", "stocks", "location", "tv",
|
||||
];
|
||||
|
||||
type Index = { bundleIds: Map<string, PebbleIconName>; prefixes: [string, PebbleIconName][]; names: Map<string, PebbleIconName> };
|
||||
let index: Index | null = null;
|
||||
|
||||
function buildIndex(): Index {
|
||||
const built: Index = { bundleIds: new Map(), prefixes: [], names: new Map() };
|
||||
for (const app of APP_ICONS) {
|
||||
for (const id of app.bundleIds) built.bundleIds.set(id.toLowerCase(), app.icon);
|
||||
for (const prefix of app.bundlePrefixes ?? []) built.prefixes.push([prefix.toLowerCase(), app.icon]);
|
||||
for (const name of app.names ?? []) built.names.set(name.toLowerCase(), app.icon);
|
||||
}
|
||||
return built;
|
||||
}
|
||||
|
||||
/**
|
||||
* The icon for a notification's source app, by bundle ID (ignoring case;
|
||||
* Apple treats bundle IDs case-insensitively), else by its display name;
|
||||
* null when the app isn't known.
|
||||
*/
|
||||
export function notificationAppIcon(bundleId: string, appName: string): PebbleIconName | null {
|
||||
index ??= buildIndex();
|
||||
const id = bundleId.trim().toLowerCase();
|
||||
if (id) {
|
||||
const byId = index.bundleIds.get(id) ?? index.prefixes.find(([prefix]) => id.startsWith(prefix))?.[1];
|
||||
if (byId) return byId;
|
||||
}
|
||||
const name = appName.trim().toLowerCase();
|
||||
return (name && index.names.get(name)) || null;
|
||||
}
|
||||
|
||||
/** The icon for an ANCS category (a CategoryID as a string); the bell for anything unknown. */
|
||||
export function notificationCategoryIcon(category: string): PebbleIconName {
|
||||
return CATEGORY_ICONS[Number(category)] ?? "generic";
|
||||
}
|
||||
|
||||
/** Every icon the table refers to, for tests. */
|
||||
export function allNotificationAppIcons(): PebbleIconName[] {
|
||||
return Array.from(new Set(APP_ICONS.map((app) => app.icon).concat(CATEGORY_ICONS)));
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
import { GrayImage } from '../graphics/image'
|
||||
import { renderIcon, type IconName } from '../graphics/icons'
|
||||
import type { GrayImage } from '../graphics/image'
|
||||
import { pebbleIcon } from '../graphics/pebble-icons'
|
||||
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
|
||||
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
|
||||
import { rememberNotificationSources } from './notification-sources'
|
||||
import type { AndroidNotification } from './notification-types'
|
||||
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
|
||||
export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons'
|
||||
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
|
||||
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff
|
||||
let active: AncsClient | null = null
|
||||
@@ -32,36 +33,32 @@ export function onAndroidNotificationPosted(listener: (key: string) => void): ()
|
||||
// (they hear every change), and an open popup closes on the repaint that follows.
|
||||
export function onAndroidNotificationRemoved(_listener: (key: string) => void): () => void { return () => {} }
|
||||
export function onAndroidNotificationsChanged(listener: () => void): () => void { return onAndroidNotificationPosted(listener) }
|
||||
// ANCS does not provide app icons, so show one for the notification's ANCS
|
||||
// category; never fetch notification sources or content from a third-party
|
||||
// icon service. Indexed by CategoryID: Other, IncomingCall, MissedCall,
|
||||
// Voicemail, Social, Schedule, Email, News, HealthAndFitness,
|
||||
// BusinessAndFinance, Location, Entertainment.
|
||||
const CATEGORY_ICONS: readonly IconName[] = ['bell','phone-incoming','phone-missed','voicemail','message-circle',
|
||||
'calendar','mail','newspaper','heart-pulse','briefcase','map-pin','tv']
|
||||
function categoryIcon(category: string): GrayImage {
|
||||
return renderIcon(CATEGORY_ICONS[Number(category)] ?? 'bell',24)?.clone() ?? bell()
|
||||
}
|
||||
// Drawn fallback for when the SVG rasterizer fails.
|
||||
function bell(): GrayImage {
|
||||
const image = new GrayImage(24,24,0)
|
||||
image.fillRoundedRect(6,5,12,13,220,5)
|
||||
image.fillRect(4,17,16,2,220)
|
||||
image.fillRect(10,20,4,2,220)
|
||||
return image
|
||||
// ANCS does not provide app icons, so show a bundled Pebble icon for the
|
||||
// source app (by bundle ID or display name), else for the notification's
|
||||
// ANCS category. Never fetch notification sources or content from a
|
||||
// third-party icon service.
|
||||
function notificationIcon(notification: AndroidNotification): GrayImage {
|
||||
const name = notificationAppIcon(notification.packageName, notification.appName) ?? notificationCategoryIcon(notification.category)
|
||||
return pebbleIcon(name).clone()
|
||||
}
|
||||
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
|
||||
// One icon per source, standing for its first notification.
|
||||
const sources = new Map<string, AndroidNotification>()
|
||||
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
|
||||
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
|
||||
return {icons: shown.map(n => categoryIcon(n.category)),keys: shown.map(n => n.key),stale:false}
|
||||
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
|
||||
}
|
||||
export function readNotificationIconByKey(key: string, _allowStale: boolean) {
|
||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key)
|
||||
return {icon: notification ? categoryIcon(notification.category) : null,stale:false}
|
||||
return {icon: notification ? notificationIcon(notification) : null,stale:false}
|
||||
}
|
||||
// ANCS carries no attachments.
|
||||
// ANCS carries no attachments, avatars or custom layouts.
|
||||
export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
|
||||
return {picture: null,stale:false}
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
export function readNotificationLargeIcon(_notification: AndroidNotification, _size: number, _allowStale: boolean) {
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
export function readNotificationCustomView(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
@@ -3,11 +3,12 @@ import { logCurrent, spanCurrent } from "./frame-timings";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { rememberNotificationSources } from "./notification-sources";
|
||||
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
|
||||
import { NOTIFICATION_ICON_SIZE } from "./notification-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
const ICON_SIZE = 24;
|
||||
const ICON_SIZE = NOTIFICATION_ICON_SIZE;
|
||||
/** Ask the native listener for every active source, including those below the list's display limit. */
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff;
|
||||
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
|
||||
@@ -23,11 +24,6 @@ let cachedKeys: string[] = [];
|
||||
let cachedAtMs = 0;
|
||||
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
|
||||
const KEYED_ICON_CACHE_MAX = 128;
|
||||
/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */
|
||||
const pictureCache = new Map<string, {
|
||||
postTime: number; maxWidth: number; maxHeight: number; picture: GrayImage | null; atMs: number;
|
||||
}>();
|
||||
const PICTURE_CACHE_MAX = 8;
|
||||
let notificationListenerProxy: any = null;
|
||||
const notificationPostedListeners = new Set<(notificationKey: string) => void>();
|
||||
const notificationRemovedListeners = new Set<(notificationKey: string) => void>();
|
||||
@@ -159,48 +155,86 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not
|
||||
return { icon: icon?.clone() ?? null, stale: false };
|
||||
}
|
||||
|
||||
export type NotificationPictureResult = {
|
||||
export type NotificationImageResult = {
|
||||
/** Shared with the cache: blit it, don't draw on it. */
|
||||
picture: GrayImage | null;
|
||||
image: GrayImage | null;
|
||||
/** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */
|
||||
stale: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Images belonging to one notification (its picture, large icon or custom
|
||||
* layout), fetched to fit a box, under the allow-stale contract of
|
||||
* readNotificationIconByKey. Cached per post of the notification: an ongoing
|
||||
* notification updated in place (Dexcom redraws its graph and reading every
|
||||
* few minutes) fires no posted event, but its postTime moves. ICON_CACHE_MS
|
||||
* is the backstop here too. Under allowStale an earlier fetch stands in, if
|
||||
* it fits.
|
||||
*/
|
||||
class NotificationImageCache {
|
||||
private readonly entries = new Map<string, {
|
||||
postTime: number; maxWidth: number; maxHeight: number; image: GrayImage | null; atMs: number;
|
||||
}>();
|
||||
|
||||
constructor(private readonly maxEntries: number, private readonly span: string) {}
|
||||
|
||||
read(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
fetch: (width: number, height: number) => GrayImage | null,
|
||||
): NotificationImageResult {
|
||||
const width = Math.round(maxWidth);
|
||||
const height = Math.round(maxHeight);
|
||||
if (!global.isAndroid || width <= 0 || height <= 0) return { image: null, stale: false };
|
||||
|
||||
const now = Date.now();
|
||||
const cached = this.entries.get(notification.key);
|
||||
if (
|
||||
cached && cached.postTime === notification.postTime && cached.maxWidth === width
|
||||
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
|
||||
) {
|
||||
return { image: cached.image, stale: false };
|
||||
}
|
||||
if (allowStale) {
|
||||
const fits = cached?.image && cached.image.width <= width && cached.image.height <= height;
|
||||
return { image: fits ? cached!.image : null, stale: true };
|
||||
}
|
||||
|
||||
const image = spanCurrent(this.span, () => fetch(width, height));
|
||||
this.entries.delete(notification.key);
|
||||
this.entries.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, image, atMs: now });
|
||||
while (this.entries.size > this.maxEntries) {
|
||||
const oldestKey = this.entries.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
this.entries.delete(oldestKey);
|
||||
}
|
||||
return { image, stale: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Pictures and custom layouts are the size of a detail view, not icons: keep just the recently viewed few. */
|
||||
const pictureCache = new NotificationImageCache(8, "fetch-notification-picture");
|
||||
const customViewCache = new NotificationImageCache(8, "fetch-notification-custom-view");
|
||||
/** Large icons show on every list card drawn, so keep a screenful and then some. */
|
||||
const largeIconCache = new NotificationImageCache(32, "fetch-notification-large-icon");
|
||||
/** Avatars and logos: photographic tone, but undithered, which at icon size reads as noise. */
|
||||
const LARGE_ICON_TONE = { gamma: PHOTO_TONE.gamma, dither: false };
|
||||
|
||||
/**
|
||||
* The picture attached to a notification (hasPicture), scaled to fit
|
||||
* maxWidth x maxHeight with the photo tone and inverted if it is drawn on a
|
||||
* light background (GrayPacket.hasLightBackground), under the allow-stale contract of
|
||||
* readNotificationIconByKey. Cached per post of the notification: an ongoing
|
||||
* notification updated in place (Dexcom redraws its graph every few minutes)
|
||||
* fires no posted event, but its postTime moves. ICON_CACHE_MS is the
|
||||
* backstop here too. Under allowStale an earlier fetch stands in, if it fits.
|
||||
* light background (GrayPacket.hasLightBackground); see NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationPicture(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
): NotificationPictureResult {
|
||||
const width = Math.round(maxWidth);
|
||||
const height = Math.round(maxHeight);
|
||||
if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) {
|
||||
return { picture: null, stale: false };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const cached = pictureCache.get(notification.key);
|
||||
if (
|
||||
cached && cached.postTime === notification.postTime && cached.maxWidth === width
|
||||
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
|
||||
) {
|
||||
return { picture: cached.picture, stale: false };
|
||||
}
|
||||
if (allowStale) {
|
||||
const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height;
|
||||
return { picture: fits ? cached!.picture : null, stale: true };
|
||||
}
|
||||
|
||||
const picture = spanCurrent("fetch-notification-picture", () =>
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasPicture) return { image: null, stale: false };
|
||||
return pictureCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
|
||||
grayImageFromPacket(
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray(
|
||||
notification.key,
|
||||
@@ -212,16 +246,56 @@ export function readNotificationPicture(
|
||||
// lit panel on the glasses: show it light-on-dark.
|
||||
true,
|
||||
),
|
||||
),
|
||||
);
|
||||
pictureCache.delete(notification.key);
|
||||
pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now });
|
||||
while (pictureCache.size > PICTURE_CACHE_MAX) {
|
||||
const oldestKey = pictureCache.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
pictureCache.delete(oldestKey);
|
||||
}
|
||||
return { picture, stale: false };
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's large icon (hasLargeIcon: a sender's avatar, a site's
|
||||
* logo), scaled to fit size x size, a light background inverted as for
|
||||
* pictures; see NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationLargeIcon(
|
||||
notification: AndroidNotification,
|
||||
size: number,
|
||||
allowStale: boolean,
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasLargeIcon) return { image: null, stale: false };
|
||||
return largeIconCache.read(notification, size, size, allowStale, (width) =>
|
||||
grayImageFromPacket(
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationLargeIconGray(
|
||||
notification.key,
|
||||
width,
|
||||
LARGE_ICON_TONE.gamma,
|
||||
LARGE_ICON_TONE.dither,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's own layout (hasCustomView) drawn as the phone's shade
|
||||
* draws it, at most a pixel per dp and trimmed to what it draws, so usually
|
||||
* smaller than maxWidth x maxHeight; dark-on-light layouts come out
|
||||
* light-on-dark. See NotificationCustomViews.renderGray and
|
||||
* NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationCustomView(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasCustomView) return { image: null, stale: false };
|
||||
return customViewCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
|
||||
grayImageFromPacket(
|
||||
// Flat UI tone: it is mostly text and icons.
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationCustomViewGray(
|
||||
notification.key,
|
||||
width,
|
||||
height,
|
||||
1,
|
||||
false,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] {
|
||||
@@ -354,6 +428,9 @@ function normalizeNotification(value: any): AndroidNotification | null {
|
||||
actions,
|
||||
foregroundService: Boolean(value.foregroundService),
|
||||
hasPicture: Boolean(value.hasPicture),
|
||||
hasLargeIcon: Boolean(value.hasLargeIcon),
|
||||
hasCustomView: Boolean(value.hasCustomView),
|
||||
customText: Array.isArray(value.customText) ? value.customText.map((text: unknown) => String(text)).filter(Boolean) : [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
/** Shared notification model; Android names are retained for existing callers. */
|
||||
|
||||
/**
|
||||
* Side of a notification's (square) source icon wherever one is drawn: the
|
||||
* status bar, notification cards and views, the Glanceboard. 25 is the size
|
||||
* of the Pebble icons iPhone notifications use (app/graphics/pebble-icons.ts),
|
||||
* which are drawn on that pixel grid; Android's icons render at any size.
|
||||
*/
|
||||
export const NOTIFICATION_ICON_SIZE = 25;
|
||||
|
||||
export type AndroidNotificationAction = { index: number; title: string; enabled: boolean };
|
||||
export type AndroidNotification = {
|
||||
key: string; packageName: string; appName: string; title: string; text: string;
|
||||
@@ -9,6 +18,15 @@ export type AndroidNotification = {
|
||||
foregroundService?: boolean;
|
||||
/** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */
|
||||
hasPicture?: boolean;
|
||||
/** Carries a large icon (Android: a sender's avatar or logo), fetched separately by readNotificationLargeIcon. */
|
||||
hasLargeIcon?: boolean;
|
||||
/** Has its own layout (Android RemoteViews), drawn by readNotificationCustomView. */
|
||||
hasCustomView?: boolean;
|
||||
/**
|
||||
* The custom layout's text, one entry per text view, when the notification has no title or
|
||||
* text of its own (Dexcom's glucose reading and units).
|
||||
*/
|
||||
customText?: string[];
|
||||
/** Full message size reported by iOS, even when the fetched text is bounded. */
|
||||
messageSize?: number;
|
||||
};
|
||||
@@ -0,0 +1,159 @@
|
||||
import { fetchTextWithUserAgent, type TextResponse } from "../util/http";
|
||||
import { normalizeOpenAiCompatibleUrl } from "./openai-chat";
|
||||
|
||||
export type DiscoveredCompatibleModel = {
|
||||
/** Exact identifier to send as model in Chat Completions. */
|
||||
id: string;
|
||||
/** Human-friendly catalog name, or id when the server supplies no name. */
|
||||
name: string;
|
||||
/** true/false when advertised; null means the server did not say. */
|
||||
supportsTools: boolean | null;
|
||||
offline: boolean;
|
||||
deprecated: boolean;
|
||||
beta: boolean;
|
||||
};
|
||||
|
||||
type DiscoveryOptions = {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
/** Bypass the short-lived successful-response cache. */
|
||||
forceRefresh?: boolean;
|
||||
};
|
||||
|
||||
type DiscoveryResponse = Pick<TextResponse, "status" | "text">;
|
||||
type DiscoveryHttp = (url: string, init: {
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
timeoutMs: number;
|
||||
dontFollowRedirects: boolean;
|
||||
}) => Promise<DiscoveryResponse>;
|
||||
type DiscoveryDependencies = {
|
||||
http?: DiscoveryHttp;
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
type CacheEntry = {
|
||||
apiKey: string;
|
||||
fetchedAt: number;
|
||||
models: DiscoveredCompatibleModel[];
|
||||
};
|
||||
|
||||
export const COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS = 10_000;
|
||||
const CACHE_TTL_MS = 60_000;
|
||||
const MAX_MODELS = 500;
|
||||
const MAX_DISCOVERY_BODY_CHARS = 2_000_000;
|
||||
const cache = new Map<string, CacheEntry>();
|
||||
|
||||
/**
|
||||
* Discover models from the configured server's own /models endpoint. No URL
|
||||
* probing, inference calls, disk persistence, or automatic model selection.
|
||||
*/
|
||||
export async function discoverOpenAiCompatibleModels(
|
||||
options: DiscoveryOptions,
|
||||
dependencies: DiscoveryDependencies = {},
|
||||
): Promise<DiscoveredCompatibleModel[]> {
|
||||
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
|
||||
if (!baseUrl) throw new Error("Set a valid compatible API URL first");
|
||||
const apiKey = options.apiKey.trim();
|
||||
const now = dependencies.now ?? Date.now;
|
||||
const cached = cache.get(baseUrl);
|
||||
if (!options.forceRefresh && cached && cached.apiKey === apiKey && now() - cached.fetchedAt < CACHE_TTL_MS) {
|
||||
return cached.models;
|
||||
}
|
||||
|
||||
const http = dependencies.http ?? defaultHttp;
|
||||
const headers: Record<string, string> = {};
|
||||
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
||||
let response: DiscoveryResponse;
|
||||
try {
|
||||
response = await http(`${baseUrl}/models`, {
|
||||
method: "GET",
|
||||
headers,
|
||||
timeoutMs: COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS,
|
||||
// Do not forward a credential to a redirect target.
|
||||
dontFollowRedirects: true,
|
||||
});
|
||||
} catch {
|
||||
throw new Error("Could not reach the configured server");
|
||||
}
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
throw new Error(describeHttpError(response.status));
|
||||
}
|
||||
|
||||
let body: any;
|
||||
let rawBody: string;
|
||||
try {
|
||||
rawBody = await response.text();
|
||||
if (rawBody.length > MAX_DISCOVERY_BODY_CHARS) throw new Error("too large");
|
||||
body = JSON.parse(rawBody);
|
||||
} catch {
|
||||
throw new Error("The server returned an invalid model list");
|
||||
}
|
||||
if (!body || !Array.isArray(body.data)) {
|
||||
throw new Error("The server did not return an OpenAI-compatible model list");
|
||||
}
|
||||
|
||||
const models = parseModels(body.data);
|
||||
cache.set(baseUrl, { apiKey, fetchedAt: now(), models });
|
||||
return models;
|
||||
}
|
||||
|
||||
/** Parse OpenAI data[] rows, retaining richer Venice metadata when present. */
|
||||
export function parseDiscoveredCompatibleModels(rows: any[]): DiscoveredCompatibleModel[] {
|
||||
return parseModels(rows);
|
||||
}
|
||||
|
||||
function parseModels(rows: any[]): DiscoveredCompatibleModel[] {
|
||||
const byId = new Map<string, DiscoveredCompatibleModel>();
|
||||
for (const row of rows) {
|
||||
if (!row || typeof row.id !== "string" || !row.id.trim()) continue;
|
||||
const id = row.id.trim();
|
||||
if (byId.has(id)) continue;
|
||||
const spec = row.model_spec && typeof row.model_spec === "object" ? row.model_spec : null;
|
||||
const capabilities = spec?.capabilities && typeof spec.capabilities === "object" ? spec.capabilities : null;
|
||||
const catalogName = typeof spec?.name === "string" ? spec.name.trim() : "";
|
||||
const genericName = typeof row.name === "string" ? row.name.trim() : "";
|
||||
byId.set(id, {
|
||||
id,
|
||||
name: catalogName || genericName || id,
|
||||
supportsTools: typeof capabilities?.supportsFunctionCalling === "boolean"
|
||||
? capabilities.supportsFunctionCalling
|
||||
: null,
|
||||
offline: spec?.offline === true,
|
||||
deprecated: Boolean(spec?.deprecation),
|
||||
beta: spec?.betaModel === true || spec?.beta === true,
|
||||
});
|
||||
if (byId.size >= MAX_MODELS) break;
|
||||
}
|
||||
return Array.from(byId.values()).sort((a, b) => {
|
||||
if (a.offline !== b.offline) return a.offline ? 1 : -1;
|
||||
const aTools = a.supportsTools === true ? 0 : a.supportsTools === null ? 1 : 2;
|
||||
const bTools = b.supportsTools === true ? 0 : b.supportsTools === null ? 1 : 2;
|
||||
if (aTools !== bTools) return aTools - bTools;
|
||||
return (a.name || a.id).localeCompare(b.name || b.id) || a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
|
||||
async function defaultHttp(url: string, init: {
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
timeoutMs: number;
|
||||
dontFollowRedirects: boolean;
|
||||
}): Promise<DiscoveryResponse> {
|
||||
return fetchTextWithUserAgent(url, init);
|
||||
}
|
||||
|
||||
function describeHttpError(status: number): string {
|
||||
if (status === 401) return "The server rejected the API key (HTTP 401)";
|
||||
if (status === 403) return "The API key cannot list these models (HTTP 403)";
|
||||
if (status === 404) return "Model discovery is not supported at this URL (HTTP 404)";
|
||||
if (status === 429) return "The server is rate limiting requests (HTTP 429)";
|
||||
if (status >= 500) return `The server is unavailable (HTTP ${status})`;
|
||||
if (status >= 300 && status < 400) return `The server redirected the request; use its direct API URL (HTTP ${status})`;
|
||||
return `Model discovery failed (HTTP ${status})`;
|
||||
}
|
||||
|
||||
/** Test hook; does not affect any saved setting. */
|
||||
export function clearCompatibleModelDiscoveryCache(): void {
|
||||
cache.clear();
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { openSseRequest, type SseListener } from "./sse";
|
||||
|
||||
/** True only for Venice's official API root, never for lookalike hosts. */
|
||||
export function isVeniceApiBaseUrl(input: string): boolean {
|
||||
return normalizeOpenAiCompatibleUrl(input) === "https://api.venice.ai/api/v1";
|
||||
}
|
||||
|
||||
/** Streaming client for OpenAI-compatible Chat Completions APIs. */
|
||||
export type OpenAiCompatibleStreamOptions = LlmStreamOptions & {
|
||||
/** Explicit API root, for example https://api.venice.ai/api/v1 or https://host/api. */
|
||||
baseUrl: string;
|
||||
/** Venice-only web search; never sent to other compatible providers. */
|
||||
webSearch?: boolean;
|
||||
};
|
||||
|
||||
type PendingToolCall = {
|
||||
id: string;
|
||||
name: string;
|
||||
inputJson: string;
|
||||
};
|
||||
|
||||
/** Remove an optional endpoint suffix, but never guess a provider's /v1 or /api prefix. */
|
||||
export function normalizeOpenAiCompatibleUrl(input: string): string {
|
||||
let url = input.trim().replace(/\/+$/, "");
|
||||
url = url.replace(/\/chat\/completions$/i, "").replace(/\/+$/, "");
|
||||
return /^https?:\/\/[^/\s?#@\\]+(?:\/[^\s?#\\]*)?$/i.test(url) ? url : "";
|
||||
}
|
||||
|
||||
export function streamOpenAiCompatibleChat(options: OpenAiCompatibleStreamOptions): LlmStreamHandle {
|
||||
const apiKey = options.apiKey.trim();
|
||||
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
|
||||
let request: { cancel(): void } | null = null;
|
||||
let settled = false;
|
||||
let text = "";
|
||||
let finishReason: string | null = null;
|
||||
const toolCalls = new Map<number, PendingToolCall>();
|
||||
const providerMetadata: Record<string, any> = {};
|
||||
const reasoningDetails = new ReasoningDetailsAccumulator();
|
||||
|
||||
const fail = (message: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
request?.cancel();
|
||||
options.onError(message);
|
||||
};
|
||||
|
||||
const done = () => {
|
||||
if (settled) return;
|
||||
const hasToolCalls = toolCalls.size > 0;
|
||||
// Never execute calls from a truncated stream or a length-limited response.
|
||||
if (hasToolCalls && finishReason !== "tool_calls" && finishReason !== "stop") {
|
||||
fail("OpenAI-compatible stream ended before its tool calls completed");
|
||||
return;
|
||||
}
|
||||
if (!text && !hasToolCalls) {
|
||||
fail("OpenAI-compatible provider ended the turn without a reply");
|
||||
return;
|
||||
}
|
||||
|
||||
const content: LlmContentBlock[] = [];
|
||||
if (text) content.push({ type: "text", text });
|
||||
const details = reasoningDetails.entries();
|
||||
if (details.length) providerMetadata.reasoning_details = details;
|
||||
if (Object.keys(providerMetadata).length) {
|
||||
content.push({ type: "provider_item", provider: "openai-compatible", item: providerMetadata });
|
||||
}
|
||||
for (const index of Array.from(toolCalls.keys()).sort((a, b) => a - b)) {
|
||||
const call = toolCalls.get(index)!;
|
||||
if (!call.name) {
|
||||
fail("OpenAI-compatible provider returned a tool call without a name");
|
||||
return;
|
||||
}
|
||||
let input: any = {};
|
||||
if (call.inputJson.trim()) {
|
||||
try {
|
||||
input = JSON.parse(call.inputJson);
|
||||
} catch {
|
||||
fail("OpenAI-compatible provider returned malformed tool arguments");
|
||||
return;
|
||||
}
|
||||
}
|
||||
content.push({
|
||||
type: "tool_use",
|
||||
id: call.id || `call_${index + 1}`,
|
||||
name: call.name,
|
||||
input,
|
||||
});
|
||||
}
|
||||
|
||||
settled = true;
|
||||
request?.cancel();
|
||||
options.onDone({
|
||||
text,
|
||||
content,
|
||||
stopReason: finishReason === "content_filter" ? "refusal"
|
||||
: hasToolCalls ? "tool_use"
|
||||
: finishReason === "length" ? "max_tokens" : "end_turn",
|
||||
});
|
||||
};
|
||||
|
||||
if (!baseUrl || !options.model.trim()) {
|
||||
setTimeout(() => fail(!baseUrl
|
||||
? "Set a valid OpenAI-compatible API URL"
|
||||
: "Set an OpenAI-compatible model ID"), 0);
|
||||
return { cancel: () => { settled = true; } };
|
||||
}
|
||||
|
||||
const body: any = {
|
||||
model: options.model.trim(),
|
||||
messages: buildChatMessages(options),
|
||||
stream: true,
|
||||
};
|
||||
// Let the provider choose its default unless a one-shot feature supplies a cap.
|
||||
if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens;
|
||||
if (options.tools?.length) {
|
||||
body.tools = options.tools.map((tool) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
},
|
||||
}));
|
||||
}
|
||||
// Provider-specific parameters must never leak to another endpoint, even
|
||||
// when a saved Venice preference is still enabled.
|
||||
if (options.webSearch === true && isVeniceApiBaseUrl(baseUrl)) {
|
||||
body.venice_parameters = { enable_web_search: "on" };
|
||||
}
|
||||
// Do not send provider-specific reasoning fields by default. Compatible
|
||||
// servers vary widely; advanced reasoning controls can be added separately.
|
||||
|
||||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
||||
|
||||
const listener: SseListener = {
|
||||
onLine: (line: string) => {
|
||||
if (settled || !line.startsWith("data:")) return;
|
||||
const payload = line.slice(5).trim();
|
||||
if (!payload) return;
|
||||
if (payload === "[DONE]") {
|
||||
if (finishReason || (!toolCalls.size && text)) done();
|
||||
else fail("OpenAI-compatible stream ended before the response completed");
|
||||
return;
|
||||
}
|
||||
let event: any;
|
||||
try {
|
||||
event = JSON.parse(payload);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event?.error) {
|
||||
fail(`OpenAI-compatible: ${String(event.error.message ?? event.error)}`);
|
||||
return;
|
||||
}
|
||||
const choice = event?.choices?.[0];
|
||||
if (!choice) return; // Usage and provider-metadata chunks may have no choices.
|
||||
const delta = choice.delta ?? {};
|
||||
if (typeof delta.content === "string" && delta.content) {
|
||||
text += delta.content;
|
||||
options.onTextDelta?.(delta.content, text);
|
||||
}
|
||||
// Reasoning is not displayed. Preserve provider metadata for replay in
|
||||
// multi-turn tool loops when the service supplies it.
|
||||
if (Array.isArray(delta.reasoning_details)) reasoningDetails.add(delta.reasoning_details);
|
||||
if (delta.thought_signature !== undefined) providerMetadata.thought_signature = delta.thought_signature;
|
||||
if (Array.isArray(delta.tool_calls)) {
|
||||
delta.tool_calls.forEach((call: any, position: number) => {
|
||||
const index = Number.isInteger(call?.index) ? Number(call.index) : position;
|
||||
const pending = toolCalls.get(index) ?? { id: "", name: "", inputJson: "" };
|
||||
if (call?.id) pending.id = String(call.id);
|
||||
if (call?.function?.name) pending.name = String(call.function.name);
|
||||
if (typeof call?.function?.arguments === "string") pending.inputJson += call.function.arguments;
|
||||
toolCalls.set(index, pending);
|
||||
});
|
||||
}
|
||||
if (choice.finish_reason) finishReason = String(choice.finish_reason);
|
||||
},
|
||||
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
|
||||
onComplete: () => {
|
||||
// Some compatible servers close after the finish chunk without [DONE].
|
||||
if (finishReason) done();
|
||||
else fail("OpenAI-compatible stream ended before the response completed");
|
||||
},
|
||||
onFailure: (message: string) => fail(`OpenAI-compatible connection failed: ${String(message)}`),
|
||||
};
|
||||
|
||||
try {
|
||||
request = openSseRequest(`${baseUrl}/chat/completions`, JSON.stringify(body), headers, listener);
|
||||
} catch (error) {
|
||||
setTimeout(() => fail(`OpenAI-compatible request failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => { settled = true; } };
|
||||
}
|
||||
|
||||
return {
|
||||
cancel: () => {
|
||||
settled = true;
|
||||
try { request?.cancel(); } catch { /* ignore */ }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** String fields of a reasoning_details entry that stream in pieces. */
|
||||
const REASONING_DETAIL_TEXT_FIELDS = ["text", "summary", "data"];
|
||||
|
||||
/**
|
||||
* Rebuilds the complete reasoning_details array (OpenRouter's format) from
|
||||
* streamed deltas. Each delta carries fragments; fragments with the same
|
||||
* `index` belong to one entry, so their text, summary, and data concatenate
|
||||
* and their other fields (e.g. a signature that arrives after the text) fill
|
||||
* in. Entries without an index are kept as sent. The replayed array must
|
||||
* match what the model produced, so it is reassembled whole rather than
|
||||
* replaying only the latest chunk.
|
||||
*/
|
||||
class ReasoningDetailsAccumulator {
|
||||
private readonly ordered: Record<string, any>[] = [];
|
||||
private readonly byIndex = new Map<number, Record<string, any>>();
|
||||
|
||||
add(fragments: any[]): void {
|
||||
for (const fragment of fragments) {
|
||||
if (!fragment || typeof fragment !== "object" || Array.isArray(fragment)) continue;
|
||||
const index = Number.isInteger(fragment.index) ? Number(fragment.index) : null;
|
||||
const existing = index === null ? undefined : this.byIndex.get(index);
|
||||
if (!existing) {
|
||||
const entry = { ...fragment };
|
||||
this.ordered.push(entry);
|
||||
if (index !== null) this.byIndex.set(index, entry);
|
||||
continue;
|
||||
}
|
||||
for (const [key, value] of Object.entries(fragment)) {
|
||||
if (value === undefined || value === null) continue;
|
||||
if (REASONING_DETAIL_TEXT_FIELDS.includes(key) && typeof value === "string" && typeof existing[key] === "string") {
|
||||
existing[key] += value;
|
||||
} else if (existing[key] === undefined || existing[key] === null) {
|
||||
existing[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries(): Record<string, any>[] {
|
||||
return this.ordered;
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert neutral conversation history to Chat Completions messages. */
|
||||
function buildChatMessages(options: OpenAiCompatibleStreamOptions): any[] {
|
||||
const messages: any[] = [];
|
||||
if (options.system) messages.push({ role: "system", content: options.system });
|
||||
|
||||
for (const message of options.messages) {
|
||||
if (typeof message.content === "string") {
|
||||
messages.push({ role: message.role, content: message.content });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (message.role === "assistant") {
|
||||
let text = "";
|
||||
const toolCalls: any[] = [];
|
||||
let providerMetadata: Record<string, any> = {};
|
||||
for (const block of message.content) {
|
||||
if (block.type === "text") text += block.text;
|
||||
else if (block.type === "tool_use") {
|
||||
toolCalls.push({
|
||||
id: block.id,
|
||||
type: "function",
|
||||
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
|
||||
});
|
||||
} else if (block.type === "provider_item" && block.provider === "openai-compatible") {
|
||||
providerMetadata = { ...providerMetadata, ...block.item };
|
||||
}
|
||||
}
|
||||
if (text || toolCalls.length) {
|
||||
messages.push({
|
||||
role: "assistant",
|
||||
content: text || (toolCalls.length ? null : ""),
|
||||
...(toolCalls.length ? { tool_calls: toolCalls } : {}),
|
||||
...providerMetadata,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let pendingText = "";
|
||||
const flushText = () => {
|
||||
if (!pendingText) return;
|
||||
messages.push({ role: "user", content: pendingText });
|
||||
pendingText = "";
|
||||
};
|
||||
for (const block of message.content) {
|
||||
if (block.type === "text") pendingText += block.text;
|
||||
else if (block.type === "tool_result") {
|
||||
flushText();
|
||||
messages.push({
|
||||
role: "tool",
|
||||
tool_call_id: block.tool_use_id,
|
||||
content: block.is_error ? `Error: ${block.content}` : block.content,
|
||||
});
|
||||
}
|
||||
}
|
||||
flushText();
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
function describeHttpError(code: number, body: string): string {
|
||||
let detail = "";
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
detail = String(parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? "");
|
||||
} catch {
|
||||
// Non-JSON error body.
|
||||
}
|
||||
switch (code) {
|
||||
case 401: return "OpenAI-compatible: invalid API key";
|
||||
case 403: return "OpenAI-compatible: API key lacks permission";
|
||||
case 429: return "OpenAI-compatible: rate limited, try again shortly";
|
||||
case 500:
|
||||
case 502:
|
||||
case 503:
|
||||
case 504:
|
||||
return "OpenAI-compatible: service unavailable, try again shortly";
|
||||
default: return `OpenAI-compatible: HTTP ${code}${detail ? ` (${detail})` : ""}`;
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
function sessionConfig(language: string): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -209,7 +209,7 @@ function sessionConfig(): object {
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model: MODEL_ID },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/** Platform-neutral process resource sample, shared by resource-usage.ts and resource-usage.ios.ts. */
|
||||
|
||||
export type ResourceSample = {
|
||||
/** Monotonic clock (counting deep sleep), ms. */
|
||||
atMs: number;
|
||||
/** Cumulative CPU time of every thread in the process, ms. */
|
||||
cpuMs: number;
|
||||
threads: number;
|
||||
/** One value per entry of the platform's MEMORY_SERIES, MiB. */
|
||||
memoryMb: number[];
|
||||
/** Platform-specific details for the latest sample (peaks, limits, thermal state). */
|
||||
status: string;
|
||||
};
|
||||
|
||||
export function kbToMb(kb: number): number {
|
||||
return Number.isFinite(kb) ? Math.max(0, kb) / 1024 : 0;
|
||||
}
|
||||
|
||||
export function formatMb(value: number): string {
|
||||
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
|
||||
return `${Math.round(value)}M`;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
|
||||
|
||||
export * from "./resource-usage-types";
|
||||
|
||||
declare const FaceclawResourceUsage: any;
|
||||
|
||||
/**
|
||||
* Memory series charted by the resource monitor, headline first: the physical
|
||||
* footprint (what jetsam enforces its limit against, and what Xcode's memory
|
||||
* gauge shows), resident memory, and bytes in use across the malloc zones.
|
||||
*/
|
||||
export const MEMORY_SERIES: readonly string[] = ["footprint", "resident", "malloc"];
|
||||
|
||||
const THERMAL_STATES = ["nominal", "fair", "serious", "critical"];
|
||||
|
||||
export function sampleResourceUsage(): ResourceSample {
|
||||
const raw = JSON.parse(String(FaceclawResourceUsage.sample())) as Record<string, number | boolean | undefined>;
|
||||
const kb = (key: string) => kbToMb(Number(raw[key] ?? 0));
|
||||
const footprintMb = kb("footprintKb");
|
||||
const peakMb = kb("footprintPeakKb");
|
||||
const availableMb = kb("availableKb");
|
||||
const thermal = THERMAL_STATES[Number(raw.thermalState)];
|
||||
return {
|
||||
atMs: Number(raw.atMs),
|
||||
cpuMs: Number(raw.cpuMs ?? 0),
|
||||
threads: Number(raw.threads ?? 0),
|
||||
memoryMb: [footprintMb, kb("residentKb"), kb("mallocKb")],
|
||||
status: [
|
||||
// Footprint plus the remaining headroom is the jetsam limit. No headroom
|
||||
// means no limit applies (the simulator) or it was just exceeded.
|
||||
availableMb > 0 ? `limit ${formatMb(footprintMb + availableMb)}` : "",
|
||||
peakMb > 0 ? `peak ${formatMb(peakMb)}` : "",
|
||||
thermal ? `thermal ${thermal}` : "",
|
||||
raw.lowPower === true ? "Low Power Mode" : "",
|
||||
].filter(Boolean).join(" · "),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
|
||||
|
||||
export * from "./resource-usage-types";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Memory series charted by the resource monitor, headline first: RSS from
|
||||
* /proc/self/status (what Android records at exit), the native malloc heap,
|
||||
* and the used Java heap.
|
||||
*/
|
||||
export const MEMORY_SERIES: readonly string[] = ["RSS", "native", "Java"];
|
||||
|
||||
export function sampleResourceUsage(): ResourceSample {
|
||||
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
|
||||
const peakRssMb = kbToMb(Number(raw[7]));
|
||||
const javaCommittedMb = kbToMb(Number(raw[5]));
|
||||
return {
|
||||
atMs: Number(raw[0]),
|
||||
cpuMs: Number(raw[1]),
|
||||
memoryMb: [kbToMb(Number(raw[2])), kbToMb(Number(raw[3])), kbToMb(Number(raw[4]))],
|
||||
threads: Number(raw[6]),
|
||||
status: [
|
||||
peakRssMb > 0 ? `peak RSS ${formatMb(peakRssMb)}` : "",
|
||||
`Java heap ${formatMb(javaCommittedMb)} committed`,
|
||||
].filter(Boolean).join(" · "),
|
||||
};
|
||||
}
|
||||
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
// A hint, not a restriction: other languages are still recognized.
|
||||
...(this.options.language ? { language_hints: [this.options.language] } : {}),
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
declare const FaceclawSpeechAnalyzer: any;
|
||||
|
||||
/**
|
||||
* How long finish() waits for the last results: the model only has to settle
|
||||
* the few seconds of audio it hadn't finalized yet.
|
||||
*/
|
||||
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
|
||||
|
||||
/**
|
||||
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
|
||||
* whose last word ends within this much before that (the model's word ends
|
||||
* against the detector's level threshold) leaves nothing before the pause
|
||||
* unfinalized, so its paragraph is complete.
|
||||
*/
|
||||
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
|
||||
|
||||
type AnalyzerEvent = {
|
||||
kind: string;
|
||||
message?: string;
|
||||
locale?: string;
|
||||
text?: string;
|
||||
final?: boolean;
|
||||
start?: number | null;
|
||||
end?: number | null;
|
||||
speechEnd?: number | null;
|
||||
};
|
||||
|
||||
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
|
||||
export function speechAnalyzerSupported(): boolean {
|
||||
try {
|
||||
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On-device long-form transcription for continuous capture, wrapped like a
|
||||
* cloud provider: it buffers audio until the model is ready (downloading it on
|
||||
* first use) and gets pause commits.
|
||||
*/
|
||||
export function createSpeechAnalyzerClient(
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): ReconnectingSttClient {
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
|
||||
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
|
||||
continuous,
|
||||
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
|
||||
* The model finalizes text in its own pieces, a few seconds behind, and can't
|
||||
* be made to settle at a pause; this gathers its pieces into paragraphs ended
|
||||
* by pauses, like the cloud providers' segments. A paragraph is shown as
|
||||
* provisional text and sent as one final (paragraphBreakAfter) once a final
|
||||
* reaches its pause, and the last one as a plain final when the audio ends.
|
||||
*
|
||||
* Only finals place breaks, by when their last word ends (the model's
|
||||
* word-end times are accurate; first words' starts reach back into the
|
||||
* silence before them, and volatile results only carry the range of all audio
|
||||
* not finalized yet).
|
||||
*/
|
||||
export class SpeechAnalyzerSttClient implements CloudSttClient {
|
||||
private native: any = null;
|
||||
private samples = 0;
|
||||
private finishing = false;
|
||||
// Audio times (s) of pauses whose paragraph hasn't ended yet.
|
||||
private pauses: number[] = [];
|
||||
// Finalized text since the last paragraph break, when its last word ends,
|
||||
// and the model's provisional rest.
|
||||
private paragraph: string[] = [];
|
||||
private paragraphEnd: number | null = null;
|
||||
private volatile = "";
|
||||
|
||||
constructor(private readonly options: CloudSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.native) return;
|
||||
const native = FaceclawSpeechAnalyzer.new();
|
||||
native.eventHandler = (json: string) => {
|
||||
if (this.native === native) this.receive(JSON.parse(json));
|
||||
};
|
||||
this.native = native;
|
||||
native.startWithLanguage(this.options.language ?? "");
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (!this.native || this.finishing || !pcm.length) return;
|
||||
this.samples += pcm.length >> 1;
|
||||
const copy = pcm.slice();
|
||||
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
|
||||
}
|
||||
|
||||
/** A pause: it ends the paragraph once the words before it are final. */
|
||||
commitSegment(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
|
||||
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.finishing = true;
|
||||
this.native.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
const native = this.native;
|
||||
this.native = null;
|
||||
native?.cancel();
|
||||
}
|
||||
|
||||
private receive(event: AnalyzerEvent): void {
|
||||
switch (event.kind) {
|
||||
case "status":
|
||||
this.options.onStatus(event.message ?? "");
|
||||
break;
|
||||
case "ready":
|
||||
this.options.onStatus("Listening (on-device)...");
|
||||
this.options.onReady?.();
|
||||
break;
|
||||
case "transcript":
|
||||
this.transcript(event);
|
||||
break;
|
||||
case "finished": {
|
||||
const text = this.liveText();
|
||||
this.paragraph = [];
|
||||
this.volatile = "";
|
||||
this.stop();
|
||||
this.options.onTranscript({ text, isFinal: true });
|
||||
break;
|
||||
}
|
||||
case "error":
|
||||
this.stop();
|
||||
this.options.onError(event.message ?? "On-device transcription failed.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private transcript(event: AnalyzerEvent): void {
|
||||
const text = (event.text ?? "").trim();
|
||||
let ended = false;
|
||||
if (!event.final) {
|
||||
this.volatile = text;
|
||||
} else {
|
||||
this.volatile = "";
|
||||
if (text) {
|
||||
const end = event.speechEnd ?? event.end ?? null;
|
||||
// Words ending after a pause come after it. (One whose words run
|
||||
// across it, if the model didn't split there, breaks a little early.)
|
||||
if (end !== null && this.pauses.some((pause) => end > pause)) {
|
||||
this.pauses = this.pauses.filter((pause) => pause >= end);
|
||||
this.endParagraph();
|
||||
}
|
||||
this.paragraph.push(text);
|
||||
this.paragraphEnd = end;
|
||||
ended = this.endCompleteParagraph();
|
||||
}
|
||||
}
|
||||
// The paragraph's final already cleared the provisional text.
|
||||
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
/** End the paragraph if its last word ends just before a pause. */
|
||||
private endCompleteParagraph(): boolean {
|
||||
const end = this.paragraphEnd;
|
||||
if (!this.paragraph.length || end === null) return false;
|
||||
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
|
||||
if (!this.pauses.some(reached)) return false;
|
||||
this.pauses = this.pauses.filter((pause) => !reached(pause));
|
||||
this.endParagraph();
|
||||
return true;
|
||||
}
|
||||
|
||||
private endParagraph(): void {
|
||||
const text = this.paragraph.join(" ");
|
||||
this.paragraph = [];
|
||||
this.paragraphEnd = null;
|
||||
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
|
||||
}
|
||||
|
||||
private liveText(): string {
|
||||
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { getStringSetting, setStringSetting } from "./settings-store";
|
||||
import { buildQuery, SpotifyApiError, SpotifyClient, type SpotifyFetch, type SpotifyTokenSource } from "./spotify-web-api";
|
||||
|
||||
/**
|
||||
* Spotify sign-in: OAuth authorization code with PKCE against the user's own
|
||||
* client ID (no client secret), redirected to a loopback address that the
|
||||
* phone catches (FaceclawWebSignIn on iOS). Spotify accepts http only for
|
||||
* loopback IP literals, and custom-scheme redirects fail with "insecure
|
||||
* redirect URI" for apps created since 2025, so the redirect is
|
||||
* http://127.0.0.1 on a fixed port the user registers in the dashboard.
|
||||
*
|
||||
* The tokens are stored with the client ID they were issued to; changing the
|
||||
* client ID setting signs out.
|
||||
*/
|
||||
|
||||
export const SPOTIFY_ACCOUNTS_BASE = "https://accounts.spotify.com";
|
||||
export const SPOTIFY_REDIRECT_PORT = 43823;
|
||||
export const SPOTIFY_REDIRECT_PATH = "/spotify-callback";
|
||||
export const SPOTIFY_REDIRECT_URI = `http://127.0.0.1:${SPOTIFY_REDIRECT_PORT}${SPOTIFY_REDIRECT_PATH}`;
|
||||
export const SPOTIFY_DASHBOARD_URL = "https://developer.spotify.com/dashboard";
|
||||
|
||||
export const SPOTIFY_CLIENT_ID_KEY = "music.spotify.clientId";
|
||||
export const SPOTIFY_ACCOUNT_KEY = "music.spotify.account";
|
||||
|
||||
export const SPOTIFY_SCOPES = [
|
||||
"user-read-playback-state",
|
||||
"user-modify-playback-state",
|
||||
"user-read-currently-playing",
|
||||
"user-read-recently-played",
|
||||
"playlist-read-private",
|
||||
"playlist-read-collaborative",
|
||||
"user-library-read",
|
||||
"user-library-modify",
|
||||
"user-follow-read",
|
||||
];
|
||||
|
||||
/** Refresh this long before Spotify's stated expiry (an hour), so a token never lapses mid-request. */
|
||||
const EXPIRY_MARGIN_MS = 60_000;
|
||||
|
||||
export type SpotifyAccount = {
|
||||
clientId: string;
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
/** Epoch ms. */
|
||||
expiresAtMs: number;
|
||||
userId: string;
|
||||
displayName: string;
|
||||
/** "premium", "free", ...; playback control needs premium. */
|
||||
product: string;
|
||||
};
|
||||
|
||||
export type SpotifyTokens = { accessToken: string; refreshToken: string; expiresAtMs: number };
|
||||
|
||||
/** The phone side of a sign-in: PKCE material and the browser sheet that catches the loopback redirect. */
|
||||
export type WebSignIn = {
|
||||
/** A fresh PKCE verifier and its S256 challenge. */
|
||||
createPkce(): { verifier: string; challenge: string };
|
||||
randomState(): string;
|
||||
/** Show the provider's page; resolves to the redirect URL, rejects with "cancelled" or why it failed. */
|
||||
open(authorizeUrl: string, port: number, path: string): Promise<string>;
|
||||
cancel(): void;
|
||||
};
|
||||
|
||||
export function spotifyClientId(): string {
|
||||
return getStringSetting(SPOTIFY_CLIENT_ID_KEY, "").trim();
|
||||
}
|
||||
|
||||
export function isValidSpotifyClientId(value: string): boolean {
|
||||
return /^[0-9a-f]{32}$/i.test(value.trim());
|
||||
}
|
||||
|
||||
/** The stored sign-in, or null when signed out or signed in with another client ID. */
|
||||
export function readSpotifyAccount(): SpotifyAccount | null {
|
||||
let raw: any;
|
||||
try {
|
||||
raw = JSON.parse(getStringSetting(SPOTIFY_ACCOUNT_KEY, "") || "null");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!raw || typeof raw.refreshToken !== "string" || !raw.refreshToken) return null;
|
||||
const clientId = spotifyClientId();
|
||||
if (!clientId || raw.clientId !== clientId) return null;
|
||||
return {
|
||||
clientId,
|
||||
accessToken: String(raw.accessToken ?? ""),
|
||||
refreshToken: raw.refreshToken,
|
||||
expiresAtMs: Number(raw.expiresAtMs) || 0,
|
||||
userId: String(raw.userId ?? ""),
|
||||
displayName: String(raw.displayName ?? ""),
|
||||
product: String(raw.product ?? ""),
|
||||
};
|
||||
}
|
||||
|
||||
export function saveSpotifyAccount(account: SpotifyAccount | null): void {
|
||||
setStringSetting(SPOTIFY_ACCOUNT_KEY, account ? JSON.stringify(account) : "");
|
||||
}
|
||||
|
||||
export function spotifyAuthorizeUrl(clientId: string, codeChallenge: string, state: string): string {
|
||||
return `${SPOTIFY_ACCOUNTS_BASE}/authorize${buildQuery({
|
||||
response_type: "code",
|
||||
client_id: clientId,
|
||||
scope: SPOTIFY_SCOPES.join(" "),
|
||||
redirect_uri: SPOTIFY_REDIRECT_URI,
|
||||
state,
|
||||
code_challenge_method: "S256",
|
||||
code_challenge: codeChallenge,
|
||||
})}`;
|
||||
}
|
||||
|
||||
/** The authorization code from Spotify's redirect; throws with a message for the glasses otherwise. */
|
||||
export function parseSpotifyRedirect(url: string, expectedState: string): string {
|
||||
const params = parseQueryString(url);
|
||||
if (params.error) {
|
||||
throw new Error(params.error === "access_denied" ? "Spotify sign-in was declined." : `Spotify sign-in failed: ${params.error}`);
|
||||
}
|
||||
if (params.state !== expectedState) throw new Error("Spotify sign-in failed: the reply didn't match this sign-in. Try again.");
|
||||
if (!params.code) throw new Error("Spotify sign-in failed: no authorization code in the reply.");
|
||||
return params.code;
|
||||
}
|
||||
|
||||
/** Query parameters of a URL (not URLSearchParams, which the NativeScript runtime mishandles). */
|
||||
export function parseQueryString(url: string): Record<string, string> {
|
||||
const query = url.split("#")[0]!.split("?").slice(1).join("?");
|
||||
const params: Record<string, string> = {};
|
||||
for (const part of query.split("&")) {
|
||||
if (!part) continue;
|
||||
const separator = part.indexOf("=");
|
||||
const key = separator < 0 ? part : part.slice(0, separator);
|
||||
const value = separator < 0 ? "" : part.slice(separator + 1);
|
||||
try {
|
||||
params[decodeURIComponent(key.replace(/\+/g, " "))] = decodeURIComponent(value.replace(/\+/g, " "));
|
||||
} catch {
|
||||
// Malformed escapes: skip the parameter.
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/** POST to Spotify's token endpoint (authorization_code or refresh_token grant). */
|
||||
export async function requestSpotifyTokens(
|
||||
fetchImpl: SpotifyFetch,
|
||||
params: Record<string, string>,
|
||||
nowMs: number,
|
||||
): Promise<SpotifyTokens> {
|
||||
let status = 0;
|
||||
let body = "";
|
||||
try {
|
||||
const response = await fetchImpl(`${SPOTIFY_ACCOUNTS_BASE}/api/token`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json" },
|
||||
body: buildQuery(params).slice(1),
|
||||
});
|
||||
status = response.status;
|
||||
body = await response.text();
|
||||
} catch (error) {
|
||||
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
|
||||
}
|
||||
let parsed: any = null;
|
||||
try {
|
||||
parsed = JSON.parse(body);
|
||||
} catch {
|
||||
// Reported below.
|
||||
}
|
||||
if (status >= 200 && status < 300 && typeof parsed?.access_token === "string") {
|
||||
const expiresIn = Number(parsed.expires_in);
|
||||
return {
|
||||
accessToken: parsed.access_token,
|
||||
// Spotify may rotate the refresh token; "" keeps the current one.
|
||||
refreshToken: typeof parsed.refresh_token === "string" ? parsed.refresh_token : "",
|
||||
expiresAtMs: nowMs + (Number.isFinite(expiresIn) && expiresIn > 0 ? expiresIn : 3600) * 1000,
|
||||
};
|
||||
}
|
||||
const error = typeof parsed?.error === "string" ? parsed.error : "";
|
||||
const description = typeof parsed?.error_description === "string" ? parsed.error_description : "";
|
||||
if (error === "invalid_grant" && params.grant_type === "refresh_token") {
|
||||
throw new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, error);
|
||||
}
|
||||
if (error === "invalid_client") {
|
||||
throw new SpotifyApiError("Spotify doesn't recognize the client ID. Check it under Spotify in the Music app menu.", status, error);
|
||||
}
|
||||
throw new SpotifyApiError(`Spotify sign-in failed: ${description || error || `HTTP ${status}`}`, status, error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Access tokens for the stored account: the current one until it nears
|
||||
* expiry, then one refresh shared by every caller waiting on it. A refresh
|
||||
* Spotify rejects (revoked access) signs out.
|
||||
*/
|
||||
export class SpotifySession implements SpotifyTokenSource {
|
||||
private refreshing: Promise<string> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly fetchImpl: SpotifyFetch,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
accessToken(forceRefresh: boolean): Promise<string> {
|
||||
const account = readSpotifyAccount();
|
||||
if (!account) return Promise.reject(new SpotifyApiError("Not signed in to Spotify.", 401, "signed_out"));
|
||||
if (!forceRefresh && account.accessToken && account.expiresAtMs - EXPIRY_MARGIN_MS > this.now()) {
|
||||
return Promise.resolve(account.accessToken);
|
||||
}
|
||||
this.refreshing ??= this.refresh(account).finally(() => {
|
||||
this.refreshing = null;
|
||||
});
|
||||
return this.refreshing;
|
||||
}
|
||||
|
||||
private async refresh(account: SpotifyAccount): Promise<string> {
|
||||
let tokens: SpotifyTokens;
|
||||
try {
|
||||
tokens = await requestSpotifyTokens(this.fetchImpl, {
|
||||
grant_type: "refresh_token",
|
||||
refresh_token: account.refreshToken,
|
||||
client_id: account.clientId,
|
||||
}, this.now());
|
||||
} catch (error) {
|
||||
if (error instanceof SpotifyApiError && error.reason === "invalid_grant" && readSpotifyAccount()?.refreshToken === account.refreshToken) {
|
||||
saveSpotifyAccount(null);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// A sign-out or another sign-in while this was in flight wins.
|
||||
if (readSpotifyAccount()?.refreshToken !== account.refreshToken) {
|
||||
throw new SpotifyApiError("Spotify sign-in changed; try again.", 401, "signed_out");
|
||||
}
|
||||
saveSpotifyAccount({
|
||||
...account,
|
||||
accessToken: tokens.accessToken,
|
||||
refreshToken: tokens.refreshToken || account.refreshToken,
|
||||
expiresAtMs: tokens.expiresAtMs,
|
||||
});
|
||||
return tokens.accessToken;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole sign-in: the browser sheet, the code exchange and the profile
|
||||
* lookup; stores and returns the account. Rejects with "cancelled" when the
|
||||
* user closes the sheet.
|
||||
*/
|
||||
export async function signInToSpotify(
|
||||
fetchImpl: SpotifyFetch,
|
||||
webSignIn: WebSignIn,
|
||||
now: () => number = Date.now,
|
||||
): Promise<SpotifyAccount> {
|
||||
const clientId = spotifyClientId();
|
||||
if (!isValidSpotifyClientId(clientId)) throw new Error("Enter your Spotify client ID first.");
|
||||
const { verifier, challenge } = webSignIn.createPkce();
|
||||
const state = webSignIn.randomState();
|
||||
const redirect = await webSignIn.open(spotifyAuthorizeUrl(clientId, challenge, state), SPOTIFY_REDIRECT_PORT, SPOTIFY_REDIRECT_PATH);
|
||||
const code = parseSpotifyRedirect(redirect, state);
|
||||
const tokens = await requestSpotifyTokens(fetchImpl, {
|
||||
grant_type: "authorization_code",
|
||||
code,
|
||||
redirect_uri: SPOTIFY_REDIRECT_URI,
|
||||
client_id: clientId,
|
||||
code_verifier: verifier,
|
||||
}, now());
|
||||
const user = await new SpotifyClient(fetchImpl, { accessToken: async () => tokens.accessToken }).me();
|
||||
const account: SpotifyAccount = {
|
||||
clientId,
|
||||
accessToken: tokens.accessToken,
|
||||
refreshToken: tokens.refreshToken,
|
||||
expiresAtMs: tokens.expiresAtMs,
|
||||
userId: user.id,
|
||||
displayName: user.displayName,
|
||||
product: user.product,
|
||||
};
|
||||
if (!account.refreshToken) throw new Error("Spotify sign-in failed: no refresh token in the reply.");
|
||||
saveSpotifyAccount(account);
|
||||
return account;
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import type { MediaBrowseItem, MediaBrowserApp, MediaBrowserBridge } from "./media-types";
|
||||
import { SPOTIFY_APP_NAME, type SpotifyPlayer } from "./spotify-player";
|
||||
import type { SpotifyClient, SpotifyItem, SpotifyPlayRequest } from "./spotify-web-api";
|
||||
|
||||
const ROOT_ROWS: MediaBrowseItem[] = [
|
||||
{ mediaId: "recent", title: "Recently played", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "playlists", title: "Playlists", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "liked", title: "Liked Songs", subtitle: "", browsable: true, playable: true },
|
||||
{ mediaId: "albums", title: "Albums", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "artists", title: "Artists", subtitle: "", browsable: true, playable: false },
|
||||
];
|
||||
|
||||
/** Separates a track's URI from the context it plays in ("" for none, "liked" for Liked Songs). */
|
||||
const TRACK_CONTEXT_SEPARATOR = "|";
|
||||
|
||||
export type SpotifyLibraryDeps = {
|
||||
client: SpotifyClient;
|
||||
/** Spotify's app id on this platform, which names the library in the browse picker. */
|
||||
packageName: string;
|
||||
player: SpotifyPlayer;
|
||||
isSignedIn(): boolean;
|
||||
/** The signed-in user's id (Liked Songs' context URI names it). */
|
||||
userId(): string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The signed-in user's Spotify library as a browse tree (Web API):
|
||||
*
|
||||
* root -> recent | playlists | liked | albums | artists
|
||||
* playlist:<id>, album:<id> -> tracks (browsable and playable)
|
||||
* artist:<id> -> albums
|
||||
* track:<uri>|<context> -> plays the track within its context
|
||||
*
|
||||
* Only playlists the user owns or collaborates on can list their tracks
|
||||
* (development-mode client IDs, since February 2026); others are play-only.
|
||||
*/
|
||||
export class SpotifyLibrary implements MediaBrowserBridge {
|
||||
private connected = false;
|
||||
/** Bumped on every connect/disconnect; answers to older requests are dropped. */
|
||||
private generation = 0;
|
||||
|
||||
/** This library's row in the browse picker. */
|
||||
readonly app: MediaBrowserApp;
|
||||
|
||||
constructor(private readonly deps: SpotifyLibraryDeps) {
|
||||
this.app = { packageName: deps.packageName, serviceClass: "", appName: SPOTIFY_APP_NAME };
|
||||
}
|
||||
|
||||
listBrowsableApps(): MediaBrowserApp[] {
|
||||
return this.deps.isSignedIn() ? [this.app] : [];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
if (app.packageName !== this.app.packageName) return Promise.reject(new Error("Not a Spotify library"));
|
||||
if (!this.deps.isSignedIn()) return Promise.reject(new Error("Sign in under Spotify in the Music app menu first."));
|
||||
this.connected = true;
|
||||
return Promise.resolve("root");
|
||||
}
|
||||
|
||||
async browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
if (!this.connected) throw new Error("Not connected");
|
||||
const generation = this.generation;
|
||||
const rows = await this.list(parentId);
|
||||
if (generation !== this.generation) throw new Error("Disconnected");
|
||||
return rows;
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
if (!this.connected) return;
|
||||
const request = this.playRequest(mediaId);
|
||||
if (request) void this.deps.player.startPlayback(request);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.connected = false;
|
||||
this.generation++;
|
||||
}
|
||||
|
||||
/** What playing a browse row starts; null for rows that don't play. */
|
||||
playRequest(mediaId: string): SpotifyPlayRequest | null {
|
||||
if (mediaId === "liked") return { contextUri: this.likedContextUri() };
|
||||
const [kind, ...rest] = mediaId.split(":");
|
||||
const id = rest.join(":");
|
||||
switch (kind) {
|
||||
case "playlist":
|
||||
case "album":
|
||||
case "artist":
|
||||
return id ? { contextUri: `spotify:${kind}:${id}` } : null;
|
||||
case "track": {
|
||||
const separator = id.lastIndexOf(TRACK_CONTEXT_SEPARATOR);
|
||||
const uri = separator < 0 ? id : id.slice(0, separator);
|
||||
const context = separator < 0 ? "" : id.slice(separator + 1);
|
||||
if (!uri) return null;
|
||||
if (context === "liked") return { contextUri: this.likedContextUri(), offsetUri: uri };
|
||||
return context ? { contextUri: context, offsetUri: uri } : { uris: [uri] };
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async list(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
const client = this.deps.client;
|
||||
if (parentId === "root") return ROOT_ROWS;
|
||||
if (parentId === "recent") {
|
||||
// The history repeats tracks; keep each one's latest play.
|
||||
const seen = new Set<string>();
|
||||
return (await client.recentlyPlayed())
|
||||
.filter((played) => !seen.has(played.item.uri) && seen.add(played.item.uri))
|
||||
.map((played) => trackRow(played.item, played.contextUri));
|
||||
}
|
||||
if (parentId === "playlists") {
|
||||
const userId = this.deps.userId();
|
||||
return (await client.playlists()).map((playlist) => {
|
||||
const listable = playlist.collaborative || playlist.ownerId === userId;
|
||||
return {
|
||||
mediaId: `playlist:${playlist.id}`,
|
||||
title: playlist.name,
|
||||
subtitle: listable ? "" : playlist.ownerName,
|
||||
browsable: listable,
|
||||
playable: true,
|
||||
};
|
||||
});
|
||||
}
|
||||
if (parentId === "liked") return (await client.savedTracks()).map((item) => trackRow(item, "liked"));
|
||||
if (parentId === "albums") return (await client.savedAlbums()).map(albumRow);
|
||||
if (parentId === "artists") {
|
||||
return (await client.followedArtists()).map((artist) => ({
|
||||
mediaId: `artist:${artist.id}`,
|
||||
title: artist.name,
|
||||
subtitle: "",
|
||||
browsable: true,
|
||||
playable: true,
|
||||
}));
|
||||
}
|
||||
const [kind, ...rest] = parentId.split(":");
|
||||
const id = rest.join(":");
|
||||
if (kind === "playlist") return (await client.playlistItems(id)).map((item) => trackRow(item, `spotify:playlist:${id}`));
|
||||
if (kind === "album") return (await client.albumTracks(id)).map((item) => trackRow(item, `spotify:album:${id}`));
|
||||
if (kind === "artist") return (await client.artistAlbums(id)).map(albumRow);
|
||||
throw new Error(`Unknown Spotify library item: ${parentId}`);
|
||||
}
|
||||
|
||||
private likedContextUri(): string {
|
||||
return `spotify:user:${this.deps.userId()}:collection`;
|
||||
}
|
||||
}
|
||||
|
||||
function trackRow(item: SpotifyItem, contextUri: string): MediaBrowseItem {
|
||||
return {
|
||||
mediaId: `track:${item.uri}${contextUri ? `${TRACK_CONTEXT_SEPARATOR}${contextUri}` : ""}`,
|
||||
title: item.name,
|
||||
subtitle: item.artists,
|
||||
browsable: false,
|
||||
playable: true,
|
||||
};
|
||||
}
|
||||
|
||||
function albumRow(album: { id: string; name: string; artists: string }): MediaBrowseItem {
|
||||
return { mediaId: `album:${album.id}`, title: album.name, subtitle: album.artists, browsable: true, playable: true };
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
import {
|
||||
DEFAULT_MEDIA_STATE,
|
||||
interpolateMediaState,
|
||||
type MediaControllerBridge,
|
||||
type MediaControllerState,
|
||||
type MediaCustomAction,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import {
|
||||
SpotifyApiError,
|
||||
type SpotifyClient,
|
||||
type SpotifyDevice,
|
||||
type SpotifyItem,
|
||||
type SpotifyPlayback,
|
||||
type SpotifyPlayRequest,
|
||||
type SpotifyRepeat,
|
||||
} from "./spotify-web-api";
|
||||
|
||||
export const SPOTIFY_APP_NAME = "Spotify";
|
||||
/** Spotify's iOS bundle id: the player's packageName in the media state on iOS. */
|
||||
export const SPOTIFY_IOS_BUNDLE_ID = "com.spotify.client";
|
||||
/** Spotify's Android package: its media session's owner, which the Web API player reports too. */
|
||||
export const SPOTIFY_ANDROID_PACKAGE = "com.spotify.music";
|
||||
|
||||
/**
|
||||
* Poll intervals while something listens. The Web API has no push channel, so
|
||||
* a track change shows up within one interval (sooner at a track's end, which
|
||||
* is polled for directly); requests count against the client ID's quota.
|
||||
*/
|
||||
const POLL_PLAYING_MS = 5_000;
|
||||
const POLL_PAUSED_MS = 10_000;
|
||||
const POLL_IDLE_MS = 20_000;
|
||||
const POLL_ERROR_MS = 30_000;
|
||||
/** Re-read soon after a command: Spotify takes a moment to report its effect. */
|
||||
const SETTLE_POLL_MS = 600;
|
||||
const SETTLE_POLLS = 2;
|
||||
const NOTICE_MS = 8_000;
|
||||
/** Re-read the queue this often while the Music app shows it, besides on every track change. */
|
||||
const QUEUE_MAX_AGE_MS = 30_000;
|
||||
/** At most this often, even on track changes (and after a failed read). */
|
||||
const QUEUE_MIN_INTERVAL_MS = 5_000;
|
||||
/** Album art: the smallest of Spotify's sizes (640/300/64) at least this wide. */
|
||||
const ART_MIN_WIDTH = 128;
|
||||
|
||||
const REPEAT_NEXT: Record<SpotifyRepeat, SpotifyRepeat> = { off: "context", context: "track", track: "off" };
|
||||
const REPEAT_LABEL: Record<SpotifyRepeat, string> = { off: "Repeat: off", context: "Repeat: all", track: "Repeat: one" };
|
||||
|
||||
export type SpotifyPlayerDeps = {
|
||||
client: SpotifyClient;
|
||||
/** The state's packageName: Spotify's app id on this platform. */
|
||||
packageName: string;
|
||||
isSignedIn(): boolean;
|
||||
/** Fetch an album-art image's encoded bytes; null on failure. */
|
||||
download(url: string): Promise<Uint8Array | null>;
|
||||
/** Encoded image bytes to the shared photo tone, fitted within maxSize; null when undecodable. */
|
||||
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
|
||||
/** The phone's output volume, for playback on this phone (Spotify won't set a phone's volume). */
|
||||
systemVolume: { get(): number; set(percent: number): void };
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Spotify Connect playback (on this phone or any of the user's devices)
|
||||
* through the Web API: state from polling GET /me/player while something
|
||||
* listens, commands as API calls whose failures (no device, no Premium)
|
||||
* become a short notice in the state.
|
||||
*/
|
||||
export class SpotifyPlayer implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Sign in to Spotify from the Music app menu to show and control Spotify.";
|
||||
readonly accessLabel = "Spotify sign-in needed";
|
||||
readonly accessAction = "open the menu";
|
||||
/** Jumping skips forward through the queue, so the chosen track ends up first. */
|
||||
readonly skipRebuildsQueue = true;
|
||||
/** When Spotify was last seen playing (epoch ms); 0 = not this run. */
|
||||
lastPlayingAtMs = 0;
|
||||
|
||||
private readonly now: () => number;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private playback: SpotifyPlayback | null = null;
|
||||
private fetchedAtMs = 0;
|
||||
private pollTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private polling: Promise<void> | null = null;
|
||||
private pollAgain = false;
|
||||
private settlePolls = 0;
|
||||
private lastSignature = "";
|
||||
private notice = "";
|
||||
private noticeUntilMs = 0;
|
||||
private noticeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private upcoming: SpotifyItem[] = [];
|
||||
/** The item the queue was read with; "" = never read. */
|
||||
private queueCurrentUri = "";
|
||||
/** When the queue was last read, or a read failed. */
|
||||
private queueFetchedAtMs = 0;
|
||||
private queueLoading = false;
|
||||
private art: { url: string; bytes: Uint8Array | null; decoded: Map<number, GrayImage | null> } | null = null;
|
||||
private artVersion = 0;
|
||||
private liked: { uri: string; value: boolean | null } | null = null;
|
||||
private volumeTarget: number | null = null;
|
||||
private volumeSending = false;
|
||||
|
||||
constructor(private readonly deps: SpotifyPlayerDeps) {
|
||||
this.now = deps.now ?? Date.now;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
if (this.listeners.size === 1) this.schedulePoll(0);
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (!this.listeners.size && !this.settlePolls) this.clearPollTimer();
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const notice = this.currentNotice();
|
||||
if (!this.deps.isSignedIn()) {
|
||||
return { ...DEFAULT_MEDIA_STATE, status: "Not signed in to Spotify.", notice };
|
||||
}
|
||||
const playback = this.playback;
|
||||
if (!playback) {
|
||||
return {
|
||||
...DEFAULT_MEDIA_STATE,
|
||||
accessEnabled: true,
|
||||
packageName: this.deps.packageName,
|
||||
appName: SPOTIFY_APP_NAME,
|
||||
status: "Nothing is playing on Spotify.",
|
||||
notice,
|
||||
};
|
||||
}
|
||||
const item = playback.item;
|
||||
const restricted = playback.device?.restricted === true;
|
||||
const allowed = (action: string) => !restricted && !playback.disallows.includes(action);
|
||||
const state: MediaControllerState = {
|
||||
available: true,
|
||||
accessEnabled: true,
|
||||
packageName: this.deps.packageName,
|
||||
appName: spotifyAppName(playback.device),
|
||||
playbackState: playback.playing ? "playing" : "paused",
|
||||
title: item?.name ?? (playback.playingType === "ad" ? "Advertisement" : ""),
|
||||
artist: item?.artists ?? "",
|
||||
album: item?.album ?? "",
|
||||
positionMs: item ? playback.progressMs : -1,
|
||||
durationMs: item?.durationMs ?? -1,
|
||||
playbackSpeed: playback.playing ? 1 : 0,
|
||||
status: "",
|
||||
canPlayPause: allowed(playback.playing ? "pausing" : "resuming"),
|
||||
canSkipNext: allowed("skipping_next"),
|
||||
canSkipPrevious: allowed("skipping_prev"),
|
||||
artVersion: this.artVersion,
|
||||
customActions: restricted ? [] : this.customActions(playback),
|
||||
notice,
|
||||
};
|
||||
return interpolateMediaState(state, this.fetchedAtMs, this.now());
|
||||
}
|
||||
|
||||
async start(): Promise<void> {}
|
||||
|
||||
async stop(): Promise<void> {}
|
||||
|
||||
async playPause(): Promise<void> {
|
||||
const playback = this.playback;
|
||||
if (!playback?.playing) {
|
||||
await this.startPlayback({});
|
||||
return;
|
||||
}
|
||||
await this.command(() => this.deps.client.pause(), () => {
|
||||
playback.progressMs = this.snapshot().positionMs;
|
||||
playback.playing = false;
|
||||
this.fetchedAtMs = this.now();
|
||||
});
|
||||
}
|
||||
|
||||
async skipNext(): Promise<void> {
|
||||
await this.command(() => this.deps.client.next());
|
||||
}
|
||||
|
||||
async skipPrevious(): Promise<void> {
|
||||
await this.command(() => this.deps.client.previous());
|
||||
}
|
||||
|
||||
/** Signed in; whether anything is playing is in the state. */
|
||||
hasAccess(): boolean {
|
||||
return this.deps.isSignedIn();
|
||||
}
|
||||
|
||||
/** Sign-in happens from the Music app menu, which shows its progress. */
|
||||
async requestAccess(): Promise<boolean> {
|
||||
return this.hasAccess();
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue ids are positions in getQueue (0 = the current item). The Web API
|
||||
* has no jump, so this skips forward that many times.
|
||||
*/
|
||||
async skipToQueueItem(id: number): Promise<void> {
|
||||
const steps = Math.round(id);
|
||||
if (steps <= 0 || steps > this.upcoming.length) return;
|
||||
await this.command(async () => {
|
||||
for (let step = 0; step < steps; step++) await this.deps.client.next();
|
||||
}, () => {
|
||||
const target = this.upcoming[steps - 1];
|
||||
if (this.playback && target) {
|
||||
this.playback.item = target;
|
||||
this.playback.progressMs = 0;
|
||||
this.fetchedAtMs = this.now();
|
||||
this.upcoming = this.upcoming.slice(steps);
|
||||
this.queueCurrentUri = target.uri;
|
||||
this.itemChanged(target);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
getMediaVolumePercent(): number {
|
||||
const device = this.playback?.device ?? null;
|
||||
if (!usesDeviceVolume(device)) return this.deps.systemVolume.get();
|
||||
return this.volumeTarget ?? device!.volumePercent;
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
const device = this.playback?.device ?? null;
|
||||
const volume = Math.max(0, Math.min(100, Math.round(volumePercent)));
|
||||
if (!usesDeviceVolume(device)) {
|
||||
this.deps.systemVolume.set(volume);
|
||||
return;
|
||||
}
|
||||
// Scrolling sends a value per step: keep one request in flight and send the latest after it.
|
||||
device!.volumePercent = volume;
|
||||
this.volumeTarget = volume;
|
||||
if (!this.volumeSending) void this.sendVolume(device!.id);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
const art = this.art;
|
||||
if (!art?.bytes) return null;
|
||||
const size = Math.round(maxSize);
|
||||
if (!art.decoded.has(size)) art.decoded.set(size, this.deps.decodeArt(art.bytes, size));
|
||||
return art.decoded.get(size) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The current item, then what Spotify will play next. Read on demand (the
|
||||
* Music app asks while it paints), so nothing else pays for the request; a
|
||||
* stale list stays up, minus the items already played, until the new one
|
||||
* arrives.
|
||||
*/
|
||||
getQueue(): MediaQueueItem[] {
|
||||
const current = this.playback?.item;
|
||||
if (!current) return [];
|
||||
const age = this.now() - this.queueFetchedAtMs;
|
||||
if (!this.queueLoading && (age > QUEUE_MAX_AGE_MS || (this.queueCurrentUri !== current.uri && age > QUEUE_MIN_INTERVAL_MS))) {
|
||||
void this.loadQueue();
|
||||
}
|
||||
if (!this.queueCurrentUri) return [];
|
||||
if (this.queueCurrentUri !== current.uri) {
|
||||
const index = this.upcoming.findIndex((item) => item.uri === current.uri);
|
||||
if (index >= 0) {
|
||||
this.upcoming = this.upcoming.slice(index + 1);
|
||||
this.queueCurrentUri = current.uri;
|
||||
}
|
||||
}
|
||||
return [current, ...this.upcoming].map((item, index) => ({
|
||||
id: index,
|
||||
title: item.name,
|
||||
subtitle: item.artists,
|
||||
active: index === 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
const playback = this.playback;
|
||||
if (!playback) return;
|
||||
const client = this.deps.client;
|
||||
if (action === "shuffle") {
|
||||
const shuffle = !playback.shuffle;
|
||||
await this.command(() => client.setShuffle(shuffle), () => {
|
||||
playback.shuffle = shuffle;
|
||||
// Shuffling reorders what comes next.
|
||||
this.queueFetchedAtMs = 0;
|
||||
});
|
||||
} else if (action === "repeat") {
|
||||
const repeat = REPEAT_NEXT[playback.repeat];
|
||||
await this.command(() => client.setRepeat(repeat), () => {
|
||||
playback.repeat = repeat;
|
||||
});
|
||||
} else if (action === "like") {
|
||||
const liked = this.liked;
|
||||
if (!liked || liked.value === null) return;
|
||||
const value = !liked.value;
|
||||
await this.command(() => (value ? client.saveToLibrary([liked.uri]) : client.removeFromLibrary([liked.uri])), () => {
|
||||
if (this.liked === liked) liked.value = value;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.poll();
|
||||
}
|
||||
|
||||
/**
|
||||
* Play something (a browsed item, a search result) or, with an empty
|
||||
* request, resume. With no active device, starts on one the account has
|
||||
* open (preferring a phone). Resolves to whether it worked; a failure is
|
||||
* left as the state's notice.
|
||||
*/
|
||||
async startPlayback(request: SpotifyPlayRequest): Promise<boolean> {
|
||||
const client = this.deps.client;
|
||||
const playback = this.playback;
|
||||
const ok = await this.command(async () => {
|
||||
try {
|
||||
await client.play(request);
|
||||
} catch (error) {
|
||||
if (!(error instanceof SpotifyApiError) || error.reason !== "NO_ACTIVE_DEVICE") throw error;
|
||||
const device = pickDevice(await client.devices());
|
||||
if (!device) throw error;
|
||||
await client.play(request, device.id);
|
||||
}
|
||||
}, () => {
|
||||
this.lastPlayingAtMs = this.now();
|
||||
if (playback && this.playback === playback && !request.contextUri && !request.uris) {
|
||||
playback.playing = true;
|
||||
this.fetchedAtMs = this.now();
|
||||
}
|
||||
});
|
||||
return ok;
|
||||
}
|
||||
|
||||
/** The account changed (signed in, out, or as someone else): forget the old account's state. */
|
||||
accountChanged(): void {
|
||||
this.playback = null;
|
||||
this.upcoming = [];
|
||||
this.queueCurrentUri = "";
|
||||
this.art = null;
|
||||
this.artVersion++;
|
||||
this.liked = null;
|
||||
this.lastPlayingAtMs = 0;
|
||||
this.emit(true);
|
||||
if (this.listeners.size) this.schedulePoll(0);
|
||||
}
|
||||
|
||||
/** Show a message about a failed command in the state's notice for a few seconds. */
|
||||
showNotice(message: string): void {
|
||||
this.notice = message;
|
||||
this.noticeUntilMs = this.now() + NOTICE_MS;
|
||||
if (this.noticeTimer !== null) clearTimeout(this.noticeTimer);
|
||||
this.noticeTimer = setTimeout(() => {
|
||||
this.noticeTimer = null;
|
||||
this.emit(true);
|
||||
}, NOTICE_MS + 50);
|
||||
this.emit(true);
|
||||
}
|
||||
|
||||
private currentNotice(): string {
|
||||
return this.notice && this.now() < this.noticeUntilMs ? this.notice : "";
|
||||
}
|
||||
|
||||
private customActions(playback: SpotifyPlayback): MediaCustomAction[] {
|
||||
if (!playback.item) return [];
|
||||
const actions: MediaCustomAction[] = [];
|
||||
if (!playback.disallows.includes("toggling_shuffle")) {
|
||||
actions.push({ action: "shuffle", name: playback.shuffle ? "Shuffle: on" : "Shuffle: off" });
|
||||
}
|
||||
if (!playback.disallows.includes("toggling_repeat_context") || !playback.disallows.includes("toggling_repeat_track")) {
|
||||
actions.push({ action: "repeat", name: REPEAT_LABEL[playback.repeat] });
|
||||
}
|
||||
const liked = this.liked;
|
||||
if (playback.item.type === "track" && liked?.uri === playback.item.uri && liked.value !== null) {
|
||||
actions.push({ action: "like", name: liked.value ? "Unlike song" : "Like song" });
|
||||
}
|
||||
return actions;
|
||||
}
|
||||
|
||||
/** Run a command; on success apply its expected effect at once, then re-read to confirm. */
|
||||
private async command(run: () => Promise<void>, applied?: () => void): Promise<boolean> {
|
||||
let ok = false;
|
||||
try {
|
||||
await run();
|
||||
ok = true;
|
||||
applied?.();
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
this.showNotice(errorMessage(error));
|
||||
}
|
||||
this.settlePolls = SETTLE_POLLS;
|
||||
this.schedulePoll(SETTLE_POLL_MS);
|
||||
return ok;
|
||||
}
|
||||
|
||||
private async sendVolume(deviceId: string): Promise<void> {
|
||||
this.volumeSending = true;
|
||||
try {
|
||||
while (this.volumeTarget !== null) {
|
||||
const volume = this.volumeTarget;
|
||||
try {
|
||||
await this.deps.client.setVolume(volume, deviceId);
|
||||
} catch (error) {
|
||||
this.showNotice(errorMessage(error));
|
||||
}
|
||||
if (this.volumeTarget === volume) this.volumeTarget = null;
|
||||
}
|
||||
} finally {
|
||||
this.volumeSending = false;
|
||||
}
|
||||
}
|
||||
|
||||
private poll(): Promise<void> {
|
||||
if (this.polling) {
|
||||
this.pollAgain = true;
|
||||
return this.polling;
|
||||
}
|
||||
this.polling = this.pollOnce().finally(() => {
|
||||
this.polling = null;
|
||||
if (this.pollAgain) {
|
||||
this.pollAgain = false;
|
||||
void this.poll();
|
||||
}
|
||||
});
|
||||
return this.polling;
|
||||
}
|
||||
|
||||
private async pollOnce(): Promise<void> {
|
||||
this.clearPollTimer();
|
||||
if (!this.deps.isSignedIn()) {
|
||||
if (this.playback) {
|
||||
this.playback = null;
|
||||
this.emit(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let delay: number;
|
||||
try {
|
||||
const playback = await this.deps.client.playback();
|
||||
this.fetchedAtMs = this.now();
|
||||
this.applyPlayback(playback);
|
||||
delay = this.nextPollDelay(playback);
|
||||
} catch (error) {
|
||||
if (!(error instanceof SpotifyApiError) || error.reason !== "signed_out") {
|
||||
console.warn(`[spotify] playback read failed: ${error}`);
|
||||
}
|
||||
// Revoked access signs out (SpotifySession); the account listener resets the state.
|
||||
delay = POLL_ERROR_MS;
|
||||
}
|
||||
if (this.settlePolls > 0) {
|
||||
this.settlePolls--;
|
||||
delay = Math.min(delay, 1_500);
|
||||
}
|
||||
this.emit(false);
|
||||
this.schedulePoll(delay);
|
||||
}
|
||||
|
||||
private applyPlayback(playback: SpotifyPlayback | null): void {
|
||||
const previousUri = this.playback?.item?.uri ?? "";
|
||||
this.playback = playback;
|
||||
if (playback?.playing) this.lastPlayingAtMs = this.now();
|
||||
const item = playback?.item ?? null;
|
||||
if ((item?.uri ?? "") !== previousUri) this.itemChanged(item);
|
||||
}
|
||||
|
||||
/** A new item: fetch its art and whether it is liked. */
|
||||
private itemChanged(item: SpotifyItem | null): void {
|
||||
const url = item ? pickArtUrl(item) : "";
|
||||
if (url !== (this.art?.url ?? "")) {
|
||||
this.artVersion++;
|
||||
this.art = url ? { url, bytes: null, decoded: new Map() } : null;
|
||||
if (url) void this.loadArt(url);
|
||||
}
|
||||
this.liked = item?.type === "track" ? { uri: item.uri, value: null } : null;
|
||||
if (item?.type === "track") void this.loadLiked(item.uri);
|
||||
}
|
||||
|
||||
private async loadArt(url: string): Promise<void> {
|
||||
let bytes: Uint8Array | null = null;
|
||||
try {
|
||||
bytes = await this.deps.download(url);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] album art download failed: ${error}`);
|
||||
}
|
||||
if (!bytes?.length || this.art?.url !== url) return;
|
||||
this.art.bytes = bytes;
|
||||
this.artVersion++;
|
||||
this.emit(true);
|
||||
}
|
||||
|
||||
private async loadLiked(uri: string): Promise<void> {
|
||||
try {
|
||||
const [value] = await this.deps.client.libraryContains([uri]);
|
||||
if (this.liked?.uri !== uri) return;
|
||||
this.liked.value = Boolean(value);
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] library check failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadQueue(): Promise<void> {
|
||||
this.queueLoading = true;
|
||||
try {
|
||||
const queue = await this.deps.client.queue();
|
||||
this.upcoming = queue.upcoming;
|
||||
this.queueCurrentUri = queue.current?.uri ?? "";
|
||||
this.queueFetchedAtMs = this.now();
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] queue read failed: ${error}`);
|
||||
this.queueFetchedAtMs = this.now();
|
||||
} finally {
|
||||
this.queueLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
private nextPollDelay(playback: SpotifyPlayback | null): number {
|
||||
if (!playback) return POLL_IDLE_MS;
|
||||
if (!playback.playing) return POLL_PAUSED_MS;
|
||||
const durationMs = playback.item?.durationMs ?? -1;
|
||||
// Catch the change to the next track as it happens.
|
||||
const remaining = durationMs > 0 ? durationMs - playback.progressMs + 700 : POLL_PLAYING_MS;
|
||||
return Math.max(1_000, Math.min(POLL_PLAYING_MS, remaining));
|
||||
}
|
||||
|
||||
private schedulePoll(delayMs: number): void {
|
||||
if (!this.listeners.size && !this.settlePolls) return;
|
||||
this.clearPollTimer();
|
||||
this.pollTimer = setTimeout(() => {
|
||||
this.pollTimer = null;
|
||||
void this.poll();
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
private clearPollTimer(): void {
|
||||
if (this.pollTimer !== null) clearTimeout(this.pollTimer);
|
||||
this.pollTimer = null;
|
||||
}
|
||||
|
||||
/** Tell listeners; unforced, only when something they draw (other than the advancing position) changed. */
|
||||
private emit(force: boolean): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
const signature = JSON.stringify({ ...state, positionMs: 0 });
|
||||
if (!force && signature === this.lastSignature) return;
|
||||
this.lastSignature = signature;
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/** "Spotify", or "Spotify · <device>" when it plays elsewhere than a phone. */
|
||||
export function spotifyAppName(device: SpotifyDevice | null): string {
|
||||
if (!device?.name || device.type === "Smartphone") return SPOTIFY_APP_NAME;
|
||||
return `${SPOTIFY_APP_NAME} · ${device.name}`;
|
||||
}
|
||||
|
||||
/** Spotify sets a speaker's or computer's volume; a phone's (this one) goes through the system volume. */
|
||||
function usesDeviceVolume(device: SpotifyDevice | null): boolean {
|
||||
return Boolean(device && device.type !== "Smartphone" && device.supportsVolume && device.volumePercent >= 0 && device.id);
|
||||
}
|
||||
|
||||
/** A device to start playback on when none is active: an open one, a phone first. */
|
||||
export function pickDevice(devices: SpotifyDevice[]): SpotifyDevice | null {
|
||||
const usable = devices.filter((device) => device.id && !device.restricted);
|
||||
return usable.find((device) => device.active)
|
||||
?? usable.find((device) => device.type === "Smartphone")
|
||||
?? usable[0]
|
||||
?? null;
|
||||
}
|
||||
|
||||
function pickArtUrl(item: SpotifyItem): string {
|
||||
const images = [...item.images].sort((a, b) => a.width - b.width);
|
||||
return (images.find((image) => image.width >= ART_MIN_WIDTH) ?? images[images.length - 1])?.url ?? "";
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
return String(error);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import {
|
||||
readSpotifyAccount,
|
||||
saveSpotifyAccount,
|
||||
signInToSpotify,
|
||||
SPOTIFY_ACCOUNT_KEY,
|
||||
SPOTIFY_CLIENT_ID_KEY,
|
||||
SpotifySession,
|
||||
type SpotifyAccount,
|
||||
type WebSignIn,
|
||||
} from "./spotify-auth";
|
||||
import { SpotifyLibrary } from "./spotify-library";
|
||||
import { errorMessage, SpotifyPlayer } from "./spotify-player";
|
||||
import { SpotifyClient, type SpotifyFetch, type SpotifySearchType } from "./spotify-web-api";
|
||||
|
||||
/** Give up on a sign-in nobody finished on the phone. */
|
||||
const SIGN_IN_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
/** What the platform supplies (spotify.ios.ts, spotify.ts). */
|
||||
export type SpotifyPlatform = {
|
||||
/** Spotify's app id on this platform (SPOTIFY_IOS_BUNDLE_ID / SPOTIFY_ANDROID_PACKAGE). */
|
||||
packageName: string;
|
||||
fetch: SpotifyFetch;
|
||||
webSignIn: WebSignIn;
|
||||
download(url: string): Promise<Uint8Array | null>;
|
||||
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
|
||||
systemVolume: { get(): number; set(percent: number): void };
|
||||
};
|
||||
|
||||
export type SpotifySignInState =
|
||||
| { phase: "idle"; error: string }
|
||||
/** The sign-in sheet is open on the phone, or waiting for Faceclaw to come to the foreground. */
|
||||
| { phase: "waiting"; error: "" };
|
||||
|
||||
/**
|
||||
* Spotify for the Music app, the media bridges and the assistant: the
|
||||
* account and its sign-in, the Web API client, the player and the library.
|
||||
*/
|
||||
export class SpotifyService {
|
||||
readonly client: SpotifyClient;
|
||||
readonly player: SpotifyPlayer;
|
||||
readonly library: SpotifyLibrary;
|
||||
private signInState: SpotifySignInState = { phase: "idle", error: "" };
|
||||
private readonly listeners = new Set<() => void>();
|
||||
private accountKey: string;
|
||||
|
||||
constructor(private readonly platform: SpotifyPlatform) {
|
||||
const isSignedIn = () => readSpotifyAccount() !== null;
|
||||
this.client = new SpotifyClient(platform.fetch, new SpotifySession(platform.fetch));
|
||||
this.player = new SpotifyPlayer({
|
||||
client: this.client,
|
||||
packageName: platform.packageName,
|
||||
isSignedIn,
|
||||
download: (url) => platform.download(url),
|
||||
decodeArt: (bytes, maxSize) => platform.decodeArt(bytes, maxSize),
|
||||
systemVolume: platform.systemVolume,
|
||||
});
|
||||
this.library = new SpotifyLibrary({
|
||||
client: this.client,
|
||||
packageName: platform.packageName,
|
||||
player: this.player,
|
||||
isSignedIn,
|
||||
userId: () => readSpotifyAccount()?.userId ?? "",
|
||||
});
|
||||
this.accountKey = accountIdentity(readSpotifyAccount());
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key !== SPOTIFY_ACCOUNT_KEY && key !== SPOTIFY_CLIENT_ID_KEY) return;
|
||||
// Token refreshes rewrite the account too; only a different sign-in resets the player.
|
||||
const identity = accountIdentity(readSpotifyAccount());
|
||||
if (identity !== this.accountKey) {
|
||||
this.accountKey = identity;
|
||||
this.player.accountChanged();
|
||||
}
|
||||
this.emit();
|
||||
});
|
||||
}
|
||||
|
||||
account(): SpotifyAccount | null {
|
||||
return readSpotifyAccount();
|
||||
}
|
||||
|
||||
signInStatus(): SpotifySignInState {
|
||||
return this.signInState;
|
||||
}
|
||||
|
||||
/** Repaint hook for the sign-in menu: account or sign-in progress changed. */
|
||||
onChange(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open Spotify's sign-in page on the phone (it waits for Faceclaw to be in
|
||||
* the foreground); the outcome lands in signInStatus().
|
||||
*/
|
||||
async signIn(): Promise<void> {
|
||||
if (this.signInState.phase === "waiting") return;
|
||||
this.signInState = { phase: "waiting", error: "" };
|
||||
this.emit();
|
||||
const timer = setTimeout(() => this.platform.webSignIn.cancel(), SIGN_IN_TIMEOUT_MS);
|
||||
try {
|
||||
await signInToSpotify(this.platform.fetch, this.platform.webSignIn);
|
||||
this.signInState = { phase: "idle", error: "" };
|
||||
} catch (error) {
|
||||
const message = errorMessage(error);
|
||||
this.signInState = { phase: "idle", error: message === "cancelled" ? "" : message };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
this.emit();
|
||||
}
|
||||
}
|
||||
|
||||
cancelSignIn(): void {
|
||||
this.platform.webSignIn.cancel();
|
||||
}
|
||||
|
||||
signOut(): void {
|
||||
saveSpotifyAccount(null);
|
||||
this.signInState = { phase: "idle", error: "" };
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Spotify and play the best match: a track, or the album, artist or
|
||||
* playlist when the request names that kind. Resolves to what started, or
|
||||
* rejects with why not.
|
||||
*/
|
||||
async playSearch(query: string, kind: SpotifySearchType = "track"): Promise<string> {
|
||||
if (!readSpotifyAccount()) throw new Error("Not signed in to Spotify. Sign in under Spotify in the Music app menu on the glasses.");
|
||||
const results = await this.client.search(query, [kind], 3);
|
||||
let request: { contextUri?: string; uris?: string[] } | null = null;
|
||||
let description = "";
|
||||
if (kind === "track" && results.tracks[0]) {
|
||||
const track = results.tracks[0];
|
||||
request = { uris: [track.uri] };
|
||||
description = `${track.name}${track.artists ? ` by ${track.artists}` : ""}`;
|
||||
} else if (kind === "album" && results.albums[0]) {
|
||||
const album = results.albums[0];
|
||||
request = { contextUri: album.uri };
|
||||
description = `the album ${album.name}${album.artists ? ` by ${album.artists}` : ""}`;
|
||||
} else if (kind === "artist" && results.artists[0]) {
|
||||
request = { contextUri: results.artists[0].uri };
|
||||
description = results.artists[0].name;
|
||||
} else if (kind === "playlist" && results.playlists[0]) {
|
||||
request = { contextUri: results.playlists[0].uri };
|
||||
description = `the playlist ${results.playlists[0].name}`;
|
||||
}
|
||||
if (!request) throw new Error(`Spotify found no ${kind} matching "${query}".`);
|
||||
if (!(await this.player.startPlayback(request))) {
|
||||
throw new Error(this.player.snapshot().notice || "Spotify didn't start playback.");
|
||||
}
|
||||
return description;
|
||||
}
|
||||
|
||||
private emit(): void {
|
||||
for (const listener of this.listeners) listener();
|
||||
}
|
||||
}
|
||||
|
||||
function accountIdentity(account: SpotifyAccount | null): string {
|
||||
return account ? `${account.clientId}/${account.userId}` : "";
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* Spotify Web API over HTTPS with the signed-in user's token: the player
|
||||
* (what is playing on any of the user's Spotify Connect devices, and control
|
||||
* of it), the user's library, and search, normalized into the small shapes
|
||||
* the Music app uses. No NativeScript imports, so the Node tests drive it with
|
||||
* a fake fetch.
|
||||
*
|
||||
* Every personal client ID is a development-mode app, which since February
|
||||
* 2026 means: the client ID's owner needs Premium, playback control needs
|
||||
* Premium, at most five allowlisted users, only owned or collaborative
|
||||
* playlists list their items, no batch lookups, and search and artist albums
|
||||
* return at most 10 results per page.
|
||||
*/
|
||||
|
||||
export const SPOTIFY_API_BASE = "https://api.spotify.com/v1";
|
||||
|
||||
const HTTP_TIMEOUT_MS = 12_000;
|
||||
/** The most a library listing gathers (a page is 50); enough to scroll through on the glasses. */
|
||||
const MAX_LIST_ITEMS = 500;
|
||||
|
||||
/** The subset of fetch this client needs (fetchTextWithUserAgent in the app). */
|
||||
export type SpotifyFetch = (
|
||||
url: string,
|
||||
init: { method: string; headers: Record<string, string>; body?: string },
|
||||
) => Promise<{ status: number; text(): Promise<string> }>;
|
||||
|
||||
/** Hands out a current access token, refreshing it when expired or when forced (after a 401). */
|
||||
export type SpotifyTokenSource = {
|
||||
accessToken(forceRefresh: boolean): Promise<string>;
|
||||
};
|
||||
|
||||
/** A failed request; `reason` is Spotify's player error code (NO_ACTIVE_DEVICE, PREMIUM_REQUIRED, ...). */
|
||||
export class SpotifyApiError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly reason = "",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SpotifyApiError";
|
||||
}
|
||||
}
|
||||
|
||||
export type SpotifyImage = { url: string; width: number; height: number };
|
||||
|
||||
export type SpotifyItem = {
|
||||
uri: string;
|
||||
type: "track" | "episode";
|
||||
name: string;
|
||||
/** Track artists, comma-joined; for an episode, its show. */
|
||||
artists: string;
|
||||
/** The track's album; for an episode, its show. "" when the API leaves it out (album track listings). */
|
||||
album: string;
|
||||
durationMs: number;
|
||||
images: SpotifyImage[];
|
||||
};
|
||||
|
||||
export type SpotifyDevice = {
|
||||
id: string;
|
||||
name: string;
|
||||
/** "Smartphone", "Computer", "Speaker", ... */
|
||||
type: string;
|
||||
active: boolean;
|
||||
/** Restricted devices accept no Web API commands. */
|
||||
restricted: boolean;
|
||||
/** -1 when the device reports none. */
|
||||
volumePercent: number;
|
||||
supportsVolume: boolean;
|
||||
};
|
||||
|
||||
export type SpotifyRepeat = "off" | "context" | "track";
|
||||
|
||||
export type SpotifyPlayback = {
|
||||
device: SpotifyDevice | null;
|
||||
playing: boolean;
|
||||
progressMs: number;
|
||||
shuffle: boolean;
|
||||
repeat: SpotifyRepeat;
|
||||
/** The playlist / album / artist playing from; "" for none (a queue of single tracks). */
|
||||
contextUri: string;
|
||||
/** null during an ad, between items, or for content the API does not describe. */
|
||||
item: SpotifyItem | null;
|
||||
/** "track", "episode", "ad" or "unknown". */
|
||||
playingType: string;
|
||||
/** The player actions the context forbids, as Spotify names them: pausing, resuming, skipping_next, ... */
|
||||
disallows: string[];
|
||||
};
|
||||
|
||||
export type SpotifyQueue = { current: SpotifyItem | null; upcoming: SpotifyItem[] };
|
||||
|
||||
export type SpotifyUser = { id: string; displayName: string; product: string };
|
||||
|
||||
export type SpotifyPlaylist = {
|
||||
id: string;
|
||||
uri: string;
|
||||
name: string;
|
||||
ownerId: string;
|
||||
ownerName: string;
|
||||
collaborative: boolean;
|
||||
total: number;
|
||||
};
|
||||
|
||||
export type SpotifyAlbum = { id: string; uri: string; name: string; artists: string };
|
||||
|
||||
export type SpotifyArtist = { id: string; uri: string; name: string };
|
||||
|
||||
/** A played track and the context it played from ("" when none). */
|
||||
export type SpotifyPlayed = { item: SpotifyItem; contextUri: string };
|
||||
|
||||
export type SpotifySearchResults = {
|
||||
tracks: SpotifyItem[];
|
||||
albums: SpotifyAlbum[];
|
||||
artists: SpotifyArtist[];
|
||||
playlists: SpotifyPlaylist[];
|
||||
};
|
||||
|
||||
export type SpotifySearchType = "track" | "album" | "artist" | "playlist";
|
||||
|
||||
/**
|
||||
* What to play: a context (playlist, album, artist, show) optionally starting
|
||||
* at one of its items, or a list of track URIs. Empty resumes the current
|
||||
* playback.
|
||||
*/
|
||||
export type SpotifyPlayRequest = {
|
||||
contextUri?: string;
|
||||
uris?: string[];
|
||||
offsetUri?: string;
|
||||
offsetPosition?: number;
|
||||
};
|
||||
|
||||
type Query = Record<string, string | number | boolean | undefined>;
|
||||
|
||||
export class SpotifyClient {
|
||||
constructor(
|
||||
private readonly fetchImpl: SpotifyFetch,
|
||||
private readonly tokens: SpotifyTokenSource,
|
||||
) {}
|
||||
|
||||
/** The current playback on whichever device is active; null when nothing is. */
|
||||
async playback(): Promise<SpotifyPlayback | null> {
|
||||
const raw = await this.request("GET", "/me/player", { query: { additional_types: "episode" } });
|
||||
return raw ? parsePlayback(raw) : null;
|
||||
}
|
||||
|
||||
async queue(): Promise<SpotifyQueue> {
|
||||
const raw = await this.request("GET", "/me/player/queue");
|
||||
return {
|
||||
current: parseItem(raw?.currently_playing),
|
||||
upcoming: asArray(raw?.queue).map(parseItem).filter(isPresent),
|
||||
};
|
||||
}
|
||||
|
||||
async devices(): Promise<SpotifyDevice[]> {
|
||||
const raw = await this.request("GET", "/me/player/devices");
|
||||
return asArray(raw?.devices).map(parseDevice).filter(isPresent);
|
||||
}
|
||||
|
||||
async play(request: SpotifyPlayRequest = {}, deviceId = ""): Promise<void> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (request.contextUri) body.context_uri = request.contextUri;
|
||||
if (request.uris?.length) body.uris = request.uris;
|
||||
if (request.offsetUri) body.offset = { uri: request.offsetUri };
|
||||
else if (request.offsetPosition !== undefined) body.offset = { position: request.offsetPosition };
|
||||
await this.request("PUT", "/me/player/play", { query: { device_id: deviceId || undefined }, body });
|
||||
}
|
||||
|
||||
async pause(): Promise<void> {
|
||||
await this.request("PUT", "/me/player/pause");
|
||||
}
|
||||
|
||||
async next(): Promise<void> {
|
||||
await this.request("POST", "/me/player/next");
|
||||
}
|
||||
|
||||
async previous(): Promise<void> {
|
||||
await this.request("POST", "/me/player/previous");
|
||||
}
|
||||
|
||||
async setShuffle(on: boolean): Promise<void> {
|
||||
await this.request("PUT", "/me/player/shuffle", { query: { state: on } });
|
||||
}
|
||||
|
||||
async setRepeat(mode: SpotifyRepeat): Promise<void> {
|
||||
await this.request("PUT", "/me/player/repeat", { query: { state: mode } });
|
||||
}
|
||||
|
||||
async setVolume(percent: number, deviceId = ""): Promise<void> {
|
||||
const volume = Math.max(0, Math.min(100, Math.round(percent)));
|
||||
await this.request("PUT", "/me/player/volume", { query: { volume_percent: volume, device_id: deviceId || undefined } });
|
||||
}
|
||||
|
||||
/** Move playback to a device (one that shows in devices() but is not active), optionally playing. */
|
||||
async transfer(deviceId: string, play: boolean): Promise<void> {
|
||||
await this.request("PUT", "/me/player", { body: { device_ids: [deviceId], play } });
|
||||
}
|
||||
|
||||
async me(): Promise<SpotifyUser> {
|
||||
const raw = await this.request("GET", "/me");
|
||||
return { id: str(raw?.id), displayName: str(raw?.display_name) || str(raw?.id), product: str(raw?.product) };
|
||||
}
|
||||
|
||||
/** Whether each URI is in the user's library (Liked Songs for tracks). */
|
||||
async libraryContains(uris: string[]): Promise<boolean[]> {
|
||||
const raw = await this.request("GET", "/me/library/contains", { query: { uris: uris.join(",") } });
|
||||
return asArray(raw).map(Boolean);
|
||||
}
|
||||
|
||||
async saveToLibrary(uris: string[]): Promise<void> {
|
||||
await this.request("PUT", "/me/library", { query: { uris: uris.join(",") } });
|
||||
}
|
||||
|
||||
async removeFromLibrary(uris: string[]): Promise<void> {
|
||||
await this.request("DELETE", "/me/library", { query: { uris: uris.join(",") } });
|
||||
}
|
||||
|
||||
playlists(): Promise<SpotifyPlaylist[]> {
|
||||
return this.collect("/me/playlists", { limit: 50 }, (raw) => asArray(raw?.items).map(parsePlaylist));
|
||||
}
|
||||
|
||||
/** Only playlists the user owns or collaborates on list their items (403 otherwise). */
|
||||
playlistItems(playlistId: string): Promise<SpotifyItem[]> {
|
||||
return this.collect(`/playlists/${encodeURIComponent(playlistId)}/items`, { limit: 50, additional_types: "track,episode" },
|
||||
(raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.item ?? entry?.track)));
|
||||
}
|
||||
|
||||
savedTracks(): Promise<SpotifyItem[]> {
|
||||
return this.collect("/me/tracks", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.track)));
|
||||
}
|
||||
|
||||
savedAlbums(): Promise<SpotifyAlbum[]> {
|
||||
return this.collect("/me/albums", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseAlbum(entry?.album)));
|
||||
}
|
||||
|
||||
albumTracks(albumId: string): Promise<SpotifyItem[]> {
|
||||
return this.collect(`/albums/${encodeURIComponent(albumId)}/tracks`, { limit: 50 }, (raw) => asArray(raw?.items).map(parseItem));
|
||||
}
|
||||
|
||||
followedArtists(): Promise<SpotifyArtist[]> {
|
||||
// Cursor-paged, with the page wrapped in an "artists" object.
|
||||
return this.collect("/me/following", { type: "artist", limit: 50 }, (raw) => asArray(raw?.artists?.items).map(parseArtist),
|
||||
(raw) => str(raw?.artists?.next));
|
||||
}
|
||||
|
||||
artistAlbums(artistId: string): Promise<SpotifyAlbum[]> {
|
||||
return this.collect(`/artists/${encodeURIComponent(artistId)}/albums`, { include_groups: "album,single,compilation", limit: 10 },
|
||||
(raw) => asArray(raw?.items).map(parseAlbum), undefined, 100);
|
||||
}
|
||||
|
||||
/** Recently played tracks, newest first (one page: the API keeps only the last 50). */
|
||||
async recentlyPlayed(): Promise<SpotifyPlayed[]> {
|
||||
const raw = await this.request("GET", "/me/player/recently-played", { query: { limit: 50 } });
|
||||
return asArray(raw?.items)
|
||||
.map((entry: any) => {
|
||||
const item = parseItem(entry?.track);
|
||||
return item ? { item, contextUri: str(entry?.context?.uri) } : null;
|
||||
})
|
||||
.filter(isPresent);
|
||||
}
|
||||
|
||||
async search(query: string, types: SpotifySearchType[], limit = 5): Promise<SpotifySearchResults> {
|
||||
const raw = await this.request("GET", "/search", {
|
||||
query: { q: query, type: types.join(","), limit: Math.max(1, Math.min(10, Math.round(limit))) },
|
||||
});
|
||||
return {
|
||||
tracks: asArray(raw?.tracks?.items).map(parseItem).filter(isPresent),
|
||||
albums: asArray(raw?.albums?.items).map(parseAlbum).filter(isPresent),
|
||||
artists: asArray(raw?.artists?.items).map(parseArtist).filter(isPresent),
|
||||
// Search results include nulls for playlists the API can no longer return.
|
||||
playlists: asArray(raw?.playlists?.items).map(parsePlaylist).filter(isPresent),
|
||||
};
|
||||
}
|
||||
|
||||
/** Gather a paged listing by following its `next` links, up to maxItems. */
|
||||
private async collect<T>(
|
||||
path: string,
|
||||
query: Query,
|
||||
parse: (raw: any) => Array<T | null>,
|
||||
nextOf: (raw: any) => string = (raw) => str(raw?.next),
|
||||
maxItems = MAX_LIST_ITEMS,
|
||||
): Promise<T[]> {
|
||||
const items: T[] = [];
|
||||
let raw = await this.request("GET", path, { query });
|
||||
for (;;) {
|
||||
for (const item of parse(raw)) if (item) items.push(item);
|
||||
const next = nextOf(raw);
|
||||
if (!next || items.length >= maxItems) return items.slice(0, maxItems);
|
||||
raw = await this.request("GET", next);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One API call. `path` is relative to the API base, or an absolute `next`
|
||||
* link from a page. Resolves to the parsed JSON body (null when empty);
|
||||
* a 401 refreshes the token and retries once.
|
||||
*/
|
||||
private async request(method: string, path: string, options: { query?: Query; body?: unknown } = {}): Promise<any> {
|
||||
const url = path.startsWith("https://") ? path : `${SPOTIFY_API_BASE}${path}${buildQuery(options.query)}`;
|
||||
const headers: Record<string, string> = { Accept: "application/json" };
|
||||
if (options.body !== undefined) headers["Content-Type"] = "application/json";
|
||||
// A command without a body still sends an empty one: Spotify answers a
|
||||
// PUT or POST with no Content-Length with 411.
|
||||
const body = options.body !== undefined ? JSON.stringify(options.body) : method === "GET" ? undefined : "";
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const authorization = `Bearer ${await this.tokens.accessToken(attempt > 0)}`;
|
||||
const response = await withTimeout(this.fetchImpl(url, { method, headers: { ...headers, Authorization: authorization }, body }));
|
||||
const text = await withTimeout(response.text());
|
||||
if (response.status === 401 && attempt === 0) continue;
|
||||
if (response.status >= 200 && response.status < 300) {
|
||||
if (!text.trim()) return null;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Commands answer 200 with a snapshot id or plain text; nothing to read.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
throw apiError(response.status, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Form / query encoding (not URLSearchParams, which the NativeScript runtime mishandles). */
|
||||
export function buildQuery(query: Query | undefined): string {
|
||||
if (!query) return "";
|
||||
const parts = Object.keys(query)
|
||||
.filter((key) => query[key] !== undefined && query[key] !== "")
|
||||
.map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(String(query[key]))}`);
|
||||
return parts.length ? `?${parts.join("&")}` : "";
|
||||
}
|
||||
|
||||
async function withTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const timeout = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new SpotifyApiError("Spotify didn't answer. Check the phone's connection.", 0)), HTTP_TIMEOUT_MS);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} catch (error) {
|
||||
if (error instanceof SpotifyApiError) throw error;
|
||||
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** A message for the glasses from Spotify's error body ({error: {status, message, reason}}). */
|
||||
function apiError(status: number, body: string): SpotifyApiError {
|
||||
let message = "";
|
||||
let reason = "";
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
message = str(parsed?.error?.message ?? parsed?.error_description);
|
||||
reason = str(parsed?.error?.reason);
|
||||
} catch {
|
||||
// Not JSON (a gateway error page).
|
||||
}
|
||||
switch (reason) {
|
||||
case "NO_ACTIVE_DEVICE":
|
||||
return new SpotifyApiError("Spotify isn't open on any device. Open Spotify on your phone, then try again.", status, reason);
|
||||
case "PREMIUM_REQUIRED":
|
||||
return new SpotifyApiError("Controlling Spotify playback needs Spotify Premium.", status, reason);
|
||||
}
|
||||
if (status === 401) return new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, reason);
|
||||
if (status === 403 && /registered|developer dashboard/i.test(message)) {
|
||||
return new SpotifyApiError("This Spotify account isn't on the client ID's user list (Developer Dashboard > User Management).", status, reason);
|
||||
}
|
||||
if (status === 429) return new SpotifyApiError("Spotify is limiting requests; try again in a minute.", status, reason);
|
||||
return new SpotifyApiError(message ? `Spotify: ${message}` : `Spotify error (HTTP ${status}).`, status, reason);
|
||||
}
|
||||
|
||||
export function parsePlayback(raw: any): SpotifyPlayback {
|
||||
const repeat = str(raw?.repeat_state);
|
||||
const disallows = raw?.actions?.disallows;
|
||||
return {
|
||||
device: parseDevice(raw?.device),
|
||||
playing: raw?.is_playing === true,
|
||||
progressMs: num(raw?.progress_ms, 0),
|
||||
shuffle: raw?.shuffle_state === true,
|
||||
repeat: repeat === "context" || repeat === "track" ? repeat : "off",
|
||||
contextUri: str(raw?.context?.uri),
|
||||
item: parseItem(raw?.item),
|
||||
playingType: str(raw?.currently_playing_type) || "unknown",
|
||||
disallows: disallows && typeof disallows === "object" ? Object.keys(disallows).filter((key) => disallows[key] === true) : [],
|
||||
};
|
||||
}
|
||||
|
||||
export function parseItem(raw: any): SpotifyItem | null {
|
||||
if (!raw || typeof raw !== "object" || typeof raw.uri !== "string" || !raw.uri) return null;
|
||||
if (raw.type === "episode") {
|
||||
const show = str(raw.show?.name);
|
||||
return {
|
||||
uri: raw.uri,
|
||||
type: "episode",
|
||||
name: str(raw.name),
|
||||
artists: show || str(raw.show?.publisher),
|
||||
album: show,
|
||||
durationMs: num(raw.duration_ms, -1),
|
||||
images: parseImages(raw.images?.length ? raw.images : raw.show?.images),
|
||||
};
|
||||
}
|
||||
return {
|
||||
uri: raw.uri,
|
||||
type: "track",
|
||||
name: str(raw.name),
|
||||
artists: joinNames(raw.artists),
|
||||
album: str(raw.album?.name),
|
||||
durationMs: num(raw.duration_ms, -1),
|
||||
images: parseImages(raw.album?.images),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseDevice(raw: any): SpotifyDevice | null {
|
||||
if (!raw || typeof raw !== "object") return null;
|
||||
const volume = raw.volume_percent;
|
||||
return {
|
||||
id: str(raw.id),
|
||||
name: str(raw.name),
|
||||
type: str(raw.type),
|
||||
active: raw.is_active === true,
|
||||
restricted: raw.is_restricted === true,
|
||||
volumePercent: typeof volume === "number" && Number.isFinite(volume) ? volume : -1,
|
||||
supportsVolume: raw.supports_volume === true,
|
||||
};
|
||||
}
|
||||
|
||||
function parsePlaylist(raw: any): SpotifyPlaylist | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return {
|
||||
id: str(raw.id),
|
||||
uri: str(raw.uri) || `spotify:playlist:${raw.id}`,
|
||||
name: str(raw.name),
|
||||
ownerId: str(raw.owner?.id),
|
||||
ownerName: str(raw.owner?.display_name) || str(raw.owner?.id),
|
||||
collaborative: raw.collaborative === true,
|
||||
// `items` replaced the deprecated `tracks` summary in 2026.
|
||||
total: num(raw.items?.total ?? raw.tracks?.total, -1),
|
||||
};
|
||||
}
|
||||
|
||||
function parseAlbum(raw: any): SpotifyAlbum | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return { id: str(raw.id), uri: str(raw.uri) || `spotify:album:${raw.id}`, name: str(raw.name), artists: joinNames(raw.artists) };
|
||||
}
|
||||
|
||||
function parseArtist(raw: any): SpotifyArtist | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return { id: str(raw.id), uri: str(raw.uri) || `spotify:artist:${raw.id}`, name: str(raw.name) };
|
||||
}
|
||||
|
||||
function parseImages(raw: any): SpotifyImage[] {
|
||||
return asArray(raw)
|
||||
.filter((image: any) => image && typeof image.url === "string" && image.url)
|
||||
.map((image: any) => ({ url: image.url, width: num(image.width, 0), height: num(image.height, 0) }));
|
||||
}
|
||||
|
||||
function joinNames(raw: any): string {
|
||||
return asArray(raw).map((entry: any) => str(entry?.name)).filter(Boolean).join(", ");
|
||||
}
|
||||
|
||||
function asArray(value: unknown): any[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function str(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function num(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
||||
}
|
||||
|
||||
function isPresent<T>(value: T | null | undefined): value is T {
|
||||
return value !== null && value !== undefined;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { fetchTextWithUserAgent } from "../util/http";
|
||||
import { downloadBinary } from "./binary-http.ios";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { fromData, toData } from "./kotlin-data";
|
||||
import type { WebSignIn } from "./spotify-auth";
|
||||
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
|
||||
import { SpotifyService } from "./spotify-service";
|
||||
|
||||
declare const FaceclawAppleMusic: any;
|
||||
declare const FaceclawCrypto: any;
|
||||
declare const FaceclawGraphics: any;
|
||||
declare const FaceclawKitIosProtocol: any;
|
||||
declare const FaceclawWebSignIn: any;
|
||||
|
||||
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Sign-in in the system browser sheet, which FaceclawWebSignIn closes once the loopback redirect arrives. */
|
||||
const webSignIn: WebSignIn = {
|
||||
createPkce: () => {
|
||||
// 64 URL-safe characters (RFC 7636 allows 43..128).
|
||||
const verifier = String(FaceclawCrypto.randomBase64Url(48));
|
||||
return { verifier, challenge: String(FaceclawCrypto.sha256Base64Url(verifier)) };
|
||||
},
|
||||
randomState: () => String(FaceclawCrypto.randomBase64Url(16)),
|
||||
open: (authorizeUrl, port, path) =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
FaceclawWebSignIn.shared().signInPortPathCompletion(authorizeUrl, port, path, (redirectUrl: string | null, error: string | null) => {
|
||||
if (redirectUrl) resolve(String(redirectUrl));
|
||||
else reject(new Error(String(error || "Sign-in failed")));
|
||||
});
|
||||
}),
|
||||
cancel: () => FaceclawWebSignIn.shared().cancel(),
|
||||
};
|
||||
|
||||
let protocol: any = null;
|
||||
let service: SpotifyService | null = null;
|
||||
|
||||
/** The Spotify service, created on first use (main thread only: it owns timers and the sign-in sheet). */
|
||||
export function spotifyService(): SpotifyService | null {
|
||||
service ??= new SpotifyService({
|
||||
packageName: SPOTIFY_IOS_BUNDLE_ID,
|
||||
fetch: (url, init) => fetchTextWithUserAgent(url, init),
|
||||
webSignIn,
|
||||
download: async (url) => {
|
||||
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
|
||||
return response.status === 200 ? response.bytes : null;
|
||||
},
|
||||
decodeArt: (bytes, maxSize) => {
|
||||
const packet = FaceclawGraphics.bgraPacketFromImageDataMaxSize(toData(bytes), Math.round(maxSize));
|
||||
if (!packet) return null;
|
||||
// The shared Kotlin tone curve and dither, as for Apple Music's art.
|
||||
protocol ??= FaceclawKitIosProtocol.new();
|
||||
return grayImageFromPacket(fromData(protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither)));
|
||||
},
|
||||
// FaceclawAppleMusic's system output volume needs no media-library access.
|
||||
systemVolume: {
|
||||
get: () => {
|
||||
const volume = Number(FaceclawAppleMusic.shared().volumePercent());
|
||||
return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1;
|
||||
},
|
||||
set: (percent) => FaceclawAppleMusic.shared().setVolumePercent(Math.max(0, Math.min(100, Math.round(percent)))),
|
||||
},
|
||||
});
|
||||
return service;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { Application, Utils } from "@nativescript/core";
|
||||
import { fetchTextWithUserAgent } from "../util/http";
|
||||
import { downloadBinary } from "./binary-http";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { copyToJavaByteBuffer } from "./java-direct-buffer";
|
||||
import type { WebSignIn } from "./spotify-auth";
|
||||
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
|
||||
import { SpotifyService } from "./spotify-service";
|
||||
|
||||
declare const android: any;
|
||||
declare const com: any;
|
||||
|
||||
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** The sign-in's listener proxy, held while Java may still call it. */
|
||||
let signInListener: unknown = null;
|
||||
|
||||
/** Faceclaw's activity is in front, so it may open the browser (Android blocks background activity starts). */
|
||||
function foregroundActivity(): any {
|
||||
return Application.suspended ? null : Application.android?.foregroundActivity ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sign-in in the phone's browser, with FaceclawWebSignIn catching the
|
||||
* loopback redirect. A sign-in started from the glasses while Faceclaw is in
|
||||
* the background opens the page when Faceclaw next comes to the front, as on
|
||||
* iOS.
|
||||
*/
|
||||
const webSignIn: WebSignIn = {
|
||||
createPkce: () => {
|
||||
// 64 URL-safe characters (RFC 7636 allows 43..128).
|
||||
const verifier = String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(48));
|
||||
return { verifier, challenge: String(com.faceclaw.app.FaceclawWebSignIn.sha256Base64Url(verifier)) };
|
||||
},
|
||||
randomState: () => String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(16)),
|
||||
open: (authorizeUrl, port, path) =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
const native = com.faceclaw.app.FaceclawWebSignIn.shared();
|
||||
let onResume: ((args: any) => void) | null = null;
|
||||
const stopWaiting = () => {
|
||||
if (onResume) Application.off(Application.resumeEvent, onResume);
|
||||
onResume = null;
|
||||
};
|
||||
const listener = new com.faceclaw.app.FaceclawWebSignInListener({
|
||||
onFinished: (redirectUrl: string | null, error: string | null) => {
|
||||
if (signInListener === listener) signInListener = null;
|
||||
stopWaiting();
|
||||
if (redirectUrl) resolve(String(redirectUrl));
|
||||
else reject(new Error(String(error || "Sign-in failed")));
|
||||
},
|
||||
});
|
||||
const startError = native.start(port, path, listener);
|
||||
if (startError) {
|
||||
reject(new Error(String(startError)));
|
||||
return;
|
||||
}
|
||||
signInListener = listener;
|
||||
const present = (activity: any) => {
|
||||
stopWaiting();
|
||||
const openError = native.openBrowser(activity, authorizeUrl);
|
||||
if (openError) native.fail(String(openError));
|
||||
};
|
||||
const activity = foregroundActivity();
|
||||
if (activity) {
|
||||
present(activity);
|
||||
return;
|
||||
}
|
||||
onResume = (args: any) => present(args?.activity ?? args?.android ?? Utils.android.getApplicationContext());
|
||||
Application.on(Application.resumeEvent, onResume);
|
||||
}),
|
||||
cancel: () => com.faceclaw.app.FaceclawWebSignIn.shared().cancel(),
|
||||
};
|
||||
|
||||
/** The phone's media-stream volume, for Spotify playing on this phone (as the session bridge reports it). */
|
||||
function audioManager(): any {
|
||||
return Utils.android.getApplicationContext()?.getSystemService(android.content.Context.AUDIO_SERVICE) ?? null;
|
||||
}
|
||||
|
||||
const systemVolume = {
|
||||
get: (): number => {
|
||||
const audio = audioManager();
|
||||
if (!audio) return -1;
|
||||
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
|
||||
if (max <= 0) return 0;
|
||||
return Math.max(0, Math.min(100, Math.round((Number(audio.getStreamVolume(android.media.AudioManager.STREAM_MUSIC)) * 100) / max)));
|
||||
},
|
||||
set: (percent: number): void => {
|
||||
const audio = audioManager();
|
||||
if (!audio) return;
|
||||
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
|
||||
const volume = Math.round((Math.max(0, Math.min(100, percent)) * max) / 100);
|
||||
audio.setStreamVolume(android.media.AudioManager.STREAM_MUSIC, volume, 0);
|
||||
},
|
||||
};
|
||||
|
||||
let service: SpotifyService | null = null;
|
||||
|
||||
/**
|
||||
* The Spotify Web API service, created on first use (main thread only: it
|
||||
* owns timers and the sign-in). Signing in is optional on Android: Spotify's
|
||||
* media session already gives now-playing and transport control, and the
|
||||
* media bridges add the Web API's extras once signed in (library browsing,
|
||||
* search, shuffle / repeat / like, playback on other Spotify devices).
|
||||
*/
|
||||
export function spotifyService(): SpotifyService | null {
|
||||
service ??= new SpotifyService({
|
||||
packageName: SPOTIFY_ANDROID_PACKAGE,
|
||||
fetch: (url, init) => fetchTextWithUserAgent(url, init),
|
||||
webSignIn,
|
||||
download: async (url) => {
|
||||
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
|
||||
return response.status === 200 ? response.bytes : null;
|
||||
},
|
||||
decodeArt: (bytes, maxSize) => {
|
||||
const size = Math.round(maxSize);
|
||||
// The shared photo tone (gamma + dither), as for media-session art.
|
||||
return grayImageFromPacket(
|
||||
com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(copyToJavaByteBuffer(bytes), size, size, PHOTO_TONE.gamma, PHOTO_TONE.dither),
|
||||
);
|
||||
},
|
||||
systemVolume,
|
||||
});
|
||||
return service;
|
||||
}
|
||||
+159
-16
@@ -2,7 +2,9 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
@@ -10,20 +12,26 @@ export type IosMicrophoneSession = {
|
||||
}
|
||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
|
||||
const RAW_RETRY_MS = 3000
|
||||
const RAW_RETRY_LIMIT = 5
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||
* the PCM capture runs open-ended, and pauses commit its segments. When the
|
||||
* session or the app lifecycle cuts one short it is drained, not discarded;
|
||||
* restarting it is the controller's call.
|
||||
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
|
||||
* is open-ended and has pauses commit its segments, so it needs a cloud
|
||||
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
|
||||
* recognizer only handles short dictations. When the session or the app
|
||||
* lifecycle cuts one short it is drained, not discarded; restarting it is the
|
||||
* controller's call.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
private readonly statuses = new Set<(state: VoiceControlState) => void>()
|
||||
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
|
||||
private readonly ends = new Set<() => void>()
|
||||
private readonly levels = new Set<(level: number) => void>()
|
||||
private status = 'Voice input ready.'
|
||||
private generation = 0
|
||||
private session: IosMicrophoneSession | null = null
|
||||
@@ -35,10 +43,13 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
|
||||
// on-device continuous capture) owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: ReconnectingSttClient | null = null
|
||||
// What transcribes the current capture, for the log.
|
||||
private recognition = 'on-device'
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||
@@ -51,6 +62,21 @@ export class IosVoiceControlBridge {
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||
// rawListeners while any owner wants it, from the session setRawSession
|
||||
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||
// it; it resumes on its own once that capture has ended.
|
||||
private rawNative: any = null
|
||||
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
|
||||
private readonly rawOwners = new Set<string>()
|
||||
private rawSession: IosMicrophoneSession | null = null
|
||||
// The session the tap enabled the mic on, while it is starting or live.
|
||||
private rawHeld: IosMicrophoneSession | null = null
|
||||
private rawLive = false
|
||||
private rawGeneration = 0
|
||||
private rawFailures = 0
|
||||
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private rawLog: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -60,15 +86,19 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
|
||||
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
this.native.pcmHandler = (pcm: NSData) => {
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
||||
// Copied: bufferFromData doesn't retain pcm, and STT clients queue chunks.
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||
this.cloud?.acceptPcm(bytes)
|
||||
if (this.continuous) this.record(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.native
|
||||
@@ -96,6 +126,8 @@ export class IosVoiceControlBridge {
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.continuous = continuous
|
||||
@@ -111,7 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -125,24 +157,34 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
this.continuous = continuous
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => this.continuous)
|
||||
const options = {
|
||||
language: voice.language,
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: (status: string) => { if (current()) this.setStatus(status) },
|
||||
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}
|
||||
const continuous = () => this.continuous
|
||||
let cloud = createCloudSttClient(voice, options, continuous)
|
||||
this.recognition = cloud ? 'cloud' : 'on-device'
|
||||
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
|
||||
if (!cloud && this.continuous && speechAnalyzerSupported()) {
|
||||
cloud = createSpeechAnalyzerClient(options, continuous)
|
||||
this.recognition = 'on-device (SpeechAnalyzer)'
|
||||
}
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
@@ -155,8 +197,11 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
|
||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||
}
|
||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||
/**
|
||||
* Whether continuous capture can run with this provider: a cloud one, or
|
||||
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
|
||||
*/
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
|
||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
@@ -222,6 +267,100 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
||||
this.stop(); this.setStatus(message); this.emitEnd()
|
||||
}
|
||||
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
|
||||
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
|
||||
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
|
||||
}
|
||||
/**
|
||||
* The glasses session the raw tap captures from (connected and unlocked),
|
||||
* or null. Owners outlive a change: the tap moves to the next session.
|
||||
*/
|
||||
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
|
||||
this.rawLog = log
|
||||
if (session === this.rawSession) return
|
||||
this.endRawTap()
|
||||
this.rawSession = session; this.rawFailures = 0
|
||||
this.syncRawTap()
|
||||
}
|
||||
/**
|
||||
* Start the decode-only raw tap on behalf of `owner`; the communicator is
|
||||
* the one setRawSession bound, not `options.communicator`. False while
|
||||
* speech capture holds the mic or no session is bound: the owner stays
|
||||
* registered and the tap starts once it can. Idempotent while running.
|
||||
*/
|
||||
startRawCapture(options: { communicator: any; owner: string }): boolean {
|
||||
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
|
||||
this.syncRawTap()
|
||||
return this.rawHeld !== null
|
||||
}
|
||||
/** `owner` no longer wants the raw tap; it stops once no owner does. */
|
||||
stopRawCapture(owner: string): void {
|
||||
if (this.rawOwners.delete(owner)) this.syncRawTap()
|
||||
}
|
||||
/** Whether the raw tap's glasses mic enable went through and still holds. */
|
||||
isRawCaptureLive(): boolean { return this.rawLive }
|
||||
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
|
||||
private syncRawTap(): void {
|
||||
const session = this.rawSession
|
||||
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
|
||||
if (this.rawHeld || this.rawRetryTimer !== null) return
|
||||
void this.beginRawTap(session)
|
||||
}
|
||||
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
|
||||
const generation = ++this.rawGeneration
|
||||
const native = this.ensureRawNative()
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
|
||||
if (error) { this.failRawTap(error); return }
|
||||
this.rawHeld = session
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.rawGeneration) return
|
||||
const copy = new Uint8Array(packet)
|
||||
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.rawGeneration) return
|
||||
this.rawLive = true
|
||||
this.rawLog('EvenHub mic: glasses microphone enabled')
|
||||
} catch (failure) {
|
||||
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
|
||||
}
|
||||
}
|
||||
private ensureRawNative(): any {
|
||||
if (!this.rawNative) {
|
||||
this.rawNative = FaceclawSpeech.new()
|
||||
// A PCM capture only ends on its own when decoding failed or fell behind.
|
||||
this.rawNative.eventHandler = (json: string) => {
|
||||
const event = JSON.parse(json)
|
||||
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
|
||||
}
|
||||
this.rawNative.pcmHandler = (pcm: NSData) => {
|
||||
// Copied: bufferFromData doesn't retain pcm.
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||
// Audio is flowing, so a later failure is a new problem worth retrying.
|
||||
this.rawFailures = 0
|
||||
for (const fn of [...this.rawListeners]) fn(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.rawNative
|
||||
}
|
||||
/** Drop the tap and retry it shortly, a bounded number of times. */
|
||||
private failRawTap(message: string): void {
|
||||
this.rawLog(`EvenHub mic: ${message}`)
|
||||
this.endRawTap()
|
||||
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
|
||||
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
|
||||
}
|
||||
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
|
||||
private endRawTap(): void {
|
||||
++this.rawGeneration
|
||||
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
|
||||
this.rawRetryTimer = null
|
||||
this.rawLive = false
|
||||
this.rawNative?.cancel()
|
||||
const session = this.rawHeld; this.rawHeld = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
const session = this.session; this.session = null
|
||||
@@ -255,7 +394,11 @@ export class IosVoiceControlBridge {
|
||||
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
||||
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private emitEnd(): void {
|
||||
for (const fn of [...this.ends]) fn()
|
||||
// After the listeners: one may start the next capture, which keeps the mic.
|
||||
this.syncRawTap()
|
||||
}
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
|
||||
// stopped for its duration.
|
||||
private readonly rawPcmListeners = new Set<RawPcmListener>();
|
||||
private readonly frameMetaListeners = new Set<FrameMetaListener>();
|
||||
private readonly micLevelListeners = new Set<(level: number) => void>();
|
||||
private rawActive = false;
|
||||
// Who wants the raw tap. It runs while any owner does, so one owner letting
|
||||
// go (an EvenHub app backgrounding) doesn't silence another.
|
||||
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.rawPcmListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
|
||||
* each decoded audio chunk, about every 50 ms, from any capture: STT in
|
||||
* every mode, or the raw tap.
|
||||
*/
|
||||
onMicLevel(listener: (level: number) => void): () => void {
|
||||
this.micLevelListeners.add(listener);
|
||||
return () => this.micLevelListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
|
||||
onFrameMeta(listener: FrameMetaListener): () => void {
|
||||
this.frameMetaListeners.add(listener);
|
||||
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
if (this.micLevelListeners.size > 0) {
|
||||
const level = pcmLevel(bytes);
|
||||
this.micLevelListeners.forEach((listener) => listener(level));
|
||||
}
|
||||
},
|
||||
onFrameMeta: (angleDegrees: number, ssr: number) => {
|
||||
if (this.frameMetaListeners.size === 0) return;
|
||||
|
||||
Reference in new issue
Block a user