Merge branch 'main' into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-10-25 01:06:38 -07:00
commit 4f8c082b52
368 files changed
+26238 -3401

No files matched your search

+7 -1
View File
@@ -1,6 +1,6 @@
import { Http } from '@nativescript/core';
import { withUserAgent } from '../util/http';
import type { BinaryResponse } from './binary-http';
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
@@ -8,3 +8,9 @@ export async function downloadBinary(url: string, timeoutMs: number): Promise<Bi
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
}
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
export async function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
const response = await Http.request({ url, method: 'POST', headers: withUserAgent(headers), content: body, timeout: timeoutMs });
return { status: response.statusCode, text: response.content ? String(response.content.toString()) : '' };
}
+23
View File
@@ -2,6 +2,7 @@ declare const com: any;
declare const java: any;
export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number };
export type BinaryPostResponse = { status: number; text: string };
let nextDownloadSerial = 0;
@@ -35,3 +36,25 @@ export function downloadBinary(url: string, timeoutMs: number): Promise<BinaryRe
new com.faceclaw.app.FaceclawFileDownload(url, path, timeoutMs, listener).start();
});
}
/**
* POST a binary body and read the response as text. Not NativeScript's http
* module: it hands an ArrayBuffer body to Java one bridge call per byte
* (ByteBuffer.wrap(Array.from(...))), which for a ~1MB WAV upload stalls the
* main thread. FaceclawBinaryPost takes it as one direct-buffer copy instead.
*/
export function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
// Required lazily so Node tests can load this module's importers.
const { copyToJavaByteBuffer } = require("./java-direct-buffer") as typeof import("./java-direct-buffer");
const entries = Object.entries(headers);
const nativeHeaders = Array.create("java.lang.String", entries.length * 2);
entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; });
return new Promise((resolve, reject) => {
const listener = new com.faceclaw.app.FaceclawBinaryPostListener({
onResponse: (status: number, text: string | null) => resolve({ status, text: String(text ?? "") }),
onFailure: (message: string | null) => reject(new Error(String(message ?? "Request failed."))),
});
// The User-Agent comes from the okhttp client's interceptor.
new com.faceclaw.app.FaceclawBinaryPost(url, copyToJavaByteBuffer(new Uint8Array(body)), nativeHeaders, timeoutMs, listener).start();
});
}
+19
View File
@@ -0,0 +1,19 @@
import type { BuildKind } from './build-info'
declare const NSBundle: any
declare const NSProcessInfo: any
let cached: BuildKind | null = null
/**
* App Store (and TestFlight) builds carry no provisioning profile; anything
* installed from Xcode or as an ad hoc build does. Simulator builds have none
* either, so they're recognized by the simulator's environment.
*/
export function buildKind(): BuildKind {
if (cached) return cached
if (NSProcessInfo.processInfo.environment.objectForKey('SIMULATOR_DEVICE_NAME')) cached = 'debug'
else if (NSBundle.mainBundle.pathForResourceOfType('embedded', 'mobileprovision')) cached = 'self-built'
else cached = 'official'
return cached
}
+54
View File
@@ -0,0 +1,54 @@
/**
* Which kind of build this is, for analytics: an official release, or one
* compiled by somebody else. Official releases are signed with the key
* scripts/release.sh uses; anything else was built by whoever installed it.
*/
import { Utils } from "@nativescript/core";
declare const android: any;
declare const java: any;
/**
* `debug` is a debuggable build (build.sh / build_and_run.sh); `self-built`
* is a release build signed with someone else's key.
*/
export type BuildKind = "official" | "self-built" | "debug" | "unknown";
/** SHA-256 of the release signing certificate (`apksigner verify --print-certs dist/Faceclaw-*.apk`). */
const OFFICIAL_SIGNING_CERT_SHA256 = "c1efe42d74f3fc4e9a13bcfc8c99e018344a55525e5b300fa960c89431e382ad";
let cached: BuildKind | null = null;
export function buildKind(): BuildKind {
cached ??= detectBuildKind();
return cached;
}
function detectBuildKind(): BuildKind {
try {
const context = Utils.android.getApplicationContext();
const packageManager = context.getPackageManager();
const PackageManager = android.content.pm.PackageManager;
const signatures =
android.os.Build.VERSION.SDK_INT >= 28
? packageManager
.getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNING_CERTIFICATES)
.signingInfo.getApkContentsSigners()
: packageManager.getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNATURES).signatures;
for (let index = 0; index < signatures.length; index++) {
const digest = java.security.MessageDigest.getInstance("SHA-256").digest(signatures[index].toByteArray());
if (hex(digest) === OFFICIAL_SIGNING_CERT_SHA256) return "official";
}
const debuggable = (context.getApplicationInfo().flags & android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) !== 0;
return debuggable ? "debug" : "self-built";
} catch (error) {
console.warn(`build-info: could not read the signing certificate: ${error}`);
return "unknown";
}
}
function hex(bytes: any): string {
let text = "";
for (let index = 0; index < bytes.length; index++) text += ((bytes[index] & 0xff) + 0x100).toString(16).slice(1);
return text;
}
+112
View File
@@ -0,0 +1,112 @@
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control";
import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt";
/**
* The transcription-provider setting plus the keys the cloud providers need.
* Shared by the Android and iOS voice bridges.
*/
export type VoiceProviderSelection = {
provider: VoiceProviderKind;
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
type CloudProvider = {
name: string;
apiKey: (selection: VoiceProviderSelection) => string;
/** Whether the provider can run; defaults to having a key. */
configured?: (selection: VoiceProviderSelection) => boolean;
/** What to tell the user when it can't. */
missing?: string;
create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient;
finalTimeoutMs?: number;
};
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
elevenlabs: {
name: "ElevenLabs",
apiKey: (selection) => selection.elevenLabsApiKey,
create: (options) => new ElevenLabsSttClient(options),
},
soniox: {
name: "Soniox",
apiKey: (selection) => selection.sonioxApiKey,
create: (options) => new SonioxSttClient(options),
},
whisper: {
name: "OpenAI",
apiKey: (selection) => selection.openAiApiKey,
create: (options) => new OpenAiRealtimeSttClient(options),
},
"whisper-server": {
name: "Whisper server",
apiKey: (selection) => selection.whisperServerApiKey ?? "",
configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")),
missing: "No Whisper server URL set; using on-device voice.",
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
};
function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean {
return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim());
}
/**
* What a capture transcribes with: the provider, and for Whisper server the
* language it is told to expect. A running capture is shared only by holders
* asking for the same.
*/
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
return selection.provider === "whisper-server"
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
: selection.provider;
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
const provider = CLOUD_PROVIDERS[selection.provider];
return Boolean(provider && isConfigured(provider, selection));
}
/**
* The cloud provider for a capture, or null to transcribe on-device. A cloud
* provider whose API key is missing falls back to on-device (reported via
* onStatus) rather than failing the capture outright. `continuous` says
* whether the capture is open-ended (Transcribe), where pauses commit segments.
*/
export function createCloudSttClient(
selection: VoiceProviderSelection,
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): ReconnectingSttClient | null {
const provider = CLOUD_PROVIDERS[selection.provider];
if (!provider) return null;
if (!isConfigured(provider, selection)) {
options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`);
return null;
}
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
continuous,
provider.finalTimeoutMs,
);
}
+9 -3
View File
@@ -2,8 +2,9 @@
* Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper,
* Soniox).
* The voice bridge holds one of these while a cloud provider owns the
* transcript; the Java controller then only decodes LC3 to PCM and hands it
* over via acceptPcm.
* transcript; the native capture (FaceclawVoiceController on Android,
* FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over
* via acceptPcm.
*/
export type CloudSttTranscriptEvent = {
@@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any {
return javaBytes;
}
/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */
/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */
export function encodeBase64(bytes: Uint8Array): string {
if (global.isIOS) {
const copy = bytes.slice();
return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions);
}
if (!global.isAndroid) return "";
return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP));
}
+10
View File
@@ -0,0 +1,10 @@
declare const FaceclawKitConnectionCounters: any
export function drainConnectionCounters(): Record<string, number> {
const counts: Record<string, number> = {}
const parsed = JSON.parse(String(FaceclawKitConnectionCounters.shared.drain())) as Record<string, unknown>
for (const [name, count] of Object.entries(parsed)) {
if (typeof count === 'number' && count > 0) counts[name] = count
}
return counts
}
+14
View File
@@ -0,0 +1,14 @@
declare const com: any;
/**
* Connection-reliability event counts since the last call, from the shared
* Kotlin session core (ConnectionCounters.kt), which keeps them process-wide.
*/
export function drainConnectionCounters(): Record<string, number> {
const counts: Record<string, number> = {};
const parsed = JSON.parse(String(com.faceclaw.app.ConnectionCounters.drain())) as Record<string, unknown>;
for (const [name, count] of Object.entries(parsed)) {
if (typeof count === "number" && count > 0) counts[name] = count;
}
return counts;
}
+6 -9
View File
@@ -1,6 +1,5 @@
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
declare const com: any;
import { openSocket, type SocketConnection } from "./socket";
/**
* ElevenLabs realtime speech-to-text over WebSocket.
@@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
export type ElevenLabsSttOptions = CloudSttOptions;
export class ElevenLabsSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// PCM that arrived before the socket finished opening; flushed on open.
@@ -36,7 +34,7 @@ export class ElevenLabsSttClient {
start(): void {
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
this.open = true;
@@ -56,12 +54,12 @@ export class ElevenLabsSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
},
});
};
try {
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
// The API key rides an xi-api-key header on the handshake.
const key = this.options.apiKey;
console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`);
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
this.ws = openSocket(url, listener, { name: "xi-api-key", value: key });
this.options.onStatus("Connecting to ElevenLabs...");
} catch (error) {
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -106,7 +104,6 @@ export class ElevenLabsSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
+32 -8
View File
@@ -141,7 +141,7 @@ class AncsListener extends NSObject {
class AudioPacketListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosAudioPacketListener];
handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void;
onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs));
}
}
@@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge {
private ancsProxy: AncsListener | null = null;
private nativeCallQueue: Promise<void> = Promise.resolve();
private queuedNativeCalls = 0;
// Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface).
private readonly nativeSurfaces = new Set<string>();
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
private readonly finishedFrameOutcomes = new Map<number, string>();
private readonly frameFinishedWaiters = new Map<number, Set<(outcome: string) => void>>();
@@ -493,13 +495,13 @@ export class FaceclawCommunicatorBridge {
});
}
/** Phone-UI preview of the current composited screen, or null if none yet. */
getCompositePreview(green = false): ImageSource | null {
/** Phone-UI preview of the current composited screen (or of `crop`'s area of it), or null if none yet. */
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
const frame = this.communicator.previewComposite();
if (!frame) return null;
const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight());
if (width <= 0 || height <= 0) return null;
return previewPixels(fromData(frame.pixels), width, height, green);
return previewPixels(fromData(frame.pixels), width, height, green, crop);
}
/** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */
@@ -522,15 +524,32 @@ export class FaceclawCommunicatorBridge {
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
const transparency = options.transparency === "color-key" ? 1 : 0;
await this.enqueueNativeCall(() => {
this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height),
Math.round(options.zOrder), transparency,
);
// Kotlin logged why; reject so callers don't treat the surface as live (as on Android,
// where the compositor's exception reaches JS).
if (configured === false) throw new Error(`Could not configure surface ${id}`);
this.nativeSurfaces.add(id);
});
}
async removeSurface(id: string): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); });
await this.enqueueNativeCall(() => {
this.nativeSurfaces.delete(id);
this.communicator.removeSurfaceId(id);
});
}
/**
* Checked inside each queued call: a render in flight when its window closed lands
* behind the removal, and Kotlin aborts on a frame for an unknown surface.
*/
private hasNativeSurface(id: string, what: string): boolean {
if (this.nativeSurfaces.has(id)) return true;
console.log(`[ios-ble] ${what} for removed surface ${id} dropped`);
return false;
}
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
@@ -545,11 +564,15 @@ export class FaceclawCommunicatorBridge {
}
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible));
});
}
async setSurfaceDepth(id: string, depth: number): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth));
});
}
async setScreenBlanked(blanked: boolean): Promise<void> {
@@ -576,6 +599,7 @@ export class FaceclawCommunicatorBridge {
const snapshot = new Uint8Array(pixels8bpp);
const draws = glyphs ? toData(new Uint8Array(glyphs)) : null;
await this.enqueueNativeCall(() => {
if (!this.hasNativeSurface(surfaceId, "frame")) return;
this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(
toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height),
fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws,
+13 -3
View File
@@ -509,11 +509,21 @@ export class FaceclawCommunicatorBridge {
/**
* Phone-UI preview of the current composited screen, or null if none yet.
* `green` tints it green-on-black (the Preview color setting) instead of
* grayscale.
* grayscale; `crop` limits it to a screen area (the Preview scaling
* setting), black where that runs past the screen's edges.
*/
getCompositePreview(green = false): ImageSource | null {
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
if (!global.isAndroid) return null;
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
const bitmap = crop
? this.communicator.getCompositePreviewBitmap(
PREVIEW_BRIGHTEN_GAMMA,
green,
Math.round(crop.x),
Math.round(crop.y),
Math.round(crop.width),
Math.round(crop.height),
)
: this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
return bitmap ? new ImageSource(bitmap) : null;
}
+19
View File
@@ -0,0 +1,19 @@
import { type ImageSource } from '@nativescript/core'
import { type GrayImage } from '../graphics/image'
import { previewPixels, whiteIconPixels } from './ios-graphics'
/** Build the phone-UI preview for a frame (brightened like Android's). */
export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
const baked = image.withDrawsBaked()
return previewPixels(baked.pixels, baked.width, baked.height)
}
/**
* A glasses icon as a white-on-transparent phone image (gray becomes alpha),
* for the dark header.
*/
export function grayImageToWhiteIconSource(image: GrayImage): ImageSource | null {
const baked = image.withDrawsBaked()
return whiteIconPixels(baked.pixels, baked.width, baked.height)
}
+136
View File
@@ -0,0 +1,136 @@
import type { LlmStreamHandle, LlmStreamOptions } from "../assistant/llm-protocol";
import { openSseRequest, type SseListener } from "./sse";
/**
* Streaming client for a Hermes Agent gateway's OpenAI-compatible API server
* (POST {baseUrl}/chat/completions). Hermes is an agent with its own tools and
* memory, so only the latest user message is sent: the X-Hermes-Session-Id header
* keeps the conversation on the server. Faceclaw's own tools are not offered.
*/
export type HermesStreamOptions = LlmStreamOptions & {
baseUrl: string;
sessionId: string;
};
/** The API root from what a user typed: adds /v1 to a bare host. Empty when unusable. */
export function normalizeHermesUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
if (!/^https?:\/\/[^/\s]+/i.test(url)) return "";
url = url.replace(/\/chat\/completions$/i, "");
if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1";
return url;
}
export function streamHermesChat(options: HermesStreamOptions): LlmStreamHandle {
let request: { cancel(): void } | null = null;
let settled = false;
let text = "";
let finishReason: string | null = null;
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
// A failed or interrupted run still ends with a finish chunk and [DONE]; the
// chunk carries an error object only when Hermes has a message to show.
if (finishReason === "error") {
fail("Hermes: the agent run failed");
return;
}
if (finishReason === "cancelled") {
fail("Hermes: the agent run was interrupted");
return;
}
if (!text) {
fail("Hermes ended the turn without a reply");
return;
}
settled = true;
request?.cancel();
options.onDone({ text, content: [{ type: "text", text }], stopReason: finishReason === "length" ? "max_tokens" : "end_turn" });
};
const baseUrl = normalizeHermesUrl(options.baseUrl);
const last = [...options.messages].reverse().find((message) => message.role === "user");
const prompt = typeof last?.content === "string" ? last.content : "";
if (!baseUrl || !prompt) {
setTimeout(() => fail(baseUrl ? "Nothing to send to Hermes" : "No Hermes URL set"), 0);
return { cancel: () => { settled = true; } };
}
const messages = [
...(options.system ? [{ role: "system", content: options.system }] : []),
{ role: "user", content: prompt },
];
const body = JSON.stringify({ model: options.model, stream: true, messages });
const headers: Record<string, string> = { "Content-Type": "application/json", "X-Hermes-Session-Id": options.sessionId };
if (options.apiKey.trim()) headers.Authorization = `Bearer ${options.apiKey.trim()}`;
const listener: SseListener = {
onLine: (line: string) => {
if (settled || !line.startsWith("data:")) return;
const payload = line.slice(5).trim();
if (payload === "[DONE]") {
done();
return;
}
let event: any;
try {
event = JSON.parse(payload);
} catch {
return;
}
if (event?.error) {
fail(`Hermes: ${String(event.error.message ?? event.error)}`);
return;
}
const choice = event?.choices?.[0];
// reasoning_content is the agent's thinking; only the answer goes to the glasses.
// Hermes opens its answer with blank lines after the reasoning; drop them.
const delta = text ? choice?.delta?.content : String(choice?.delta?.content ?? "").trimStart();
if (typeof delta === "string" && delta) {
text += delta;
options.onTextDelta?.(delta, text);
}
if (choice?.finish_reason) finishReason = String(choice.finish_reason);
},
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
// Some servers close without [DONE] after the final chunk. Hermes always sends
// a finish_reason, so a stream closed without one was cut off.
onComplete: () => (finishReason ? done() : fail(text ? "Hermes: the reply was cut off" : "Hermes stream ended without a reply")),
onFailure: (message: string) => fail(`Hermes connection failed: ${String(message)}`),
};
try {
request = openSseRequest(`${baseUrl}/chat/completions`, body, headers, listener);
} catch (error) {
setTimeout(() => fail(`Hermes request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => { settled = true; } };
}
return {
cancel: () => {
settled = true;
try {
request?.cancel();
} catch {
// ignore
}
},
};
}
function describeHttpError(code: number, body: string): string {
if (code === 401 || code === 403) return "Hermes: the API server rejected the key";
let detail = "";
try {
detail = String(JSON.parse(body)?.error?.message ?? "");
} catch {
// non-JSON error body
}
return `Hermes: HTTP ${code}${detail ? ` (${detail})` : ""}`;
}
+26 -3
View File
@@ -3,12 +3,35 @@ import { GrayImage } from '../graphics/image'
declare const FaceclawGraphics: any
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
type CropRect = { x: number; y: number; width: number; height: number }
/** The gray pixels of a screen area; black where it runs past the frame's edges. */
function cropGray(pixels: Uint8Array, width: number, height: number, crop: CropRect): Uint8Array {
const cropped = new Uint8Array(crop.width * crop.height)
const left = Math.max(crop.x, 0), right = Math.min(crop.x + crop.width, width)
if (right <= left) return cropped
for (let y = Math.max(crop.y, 0); y < Math.min(crop.y + crop.height, height); y++) {
cropped.set(pixels.subarray(y * width + left, y * width + right), (y - crop.y) * crop.width + (left - crop.x))
}
return cropped
}
/** Gray pixels as the phone mirror's image, optionally cropped to a screen area (Preview scaling). */
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false, crop: CropRect | null = null): ImageSource {
// NSData copies the buffer; the UIImage owns its pixels after this call.
const copy = crop ? cropGray(pixels, width, height, crop) : new Uint8Array(pixels)
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
const native = FaceclawGraphics.previewWidthHeightGreen(data, crop?.width ?? width, crop?.height ?? height, green)
if (!native) throw new Error('Could not create preview image')
return new ImageSource(native)
}
/** Gray pixels as a white-on-transparent image (gray becomes alpha). */
export function whiteIconPixels(pixels: Uint8Array, width: number, height: number): ImageSource {
const copy = new Uint8Array(pixels)
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
if (!native) throw new Error('Could not create preview image')
const native = FaceclawGraphics.whiteIconWidthHeight(data, width, height)
if (!native) throw new Error('Could not create icon image')
return new ImageSource(native)
}
+9
View File
@@ -5,6 +5,15 @@ export type CurrentLocation = {
timestampMs: number;
};
/** A reverse-geocoded place; parts the geocoder doesn't know are empty. */
export type Place = {
/** City or town. */
locality: string;
/** State, province or similar. */
region: string;
country: string;
};
export type TrackedLocation = {
latitude: number;
longitude: number;
+70
View File
@@ -0,0 +1,70 @@
import {
APPLE_MUSIC_APP_NAME,
APPLE_MUSIC_PACKAGE,
parseMediaBrowseItems,
type MediaBrowseItem,
type MediaBrowserApp,
type MediaBrowserBridge,
} from "./media-types";
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
declare const FaceclawAppleMusic: any;
const APPLE_MUSIC_APP: MediaBrowserApp = { packageName: APPLE_MUSIC_PACKAGE, serviceClass: "", appName: APPLE_MUSIC_APP_NAME };
/**
* The Apple Music library (playlists, artists, albums, songs in the
* on-device / iCloud music library) as a browse tree, through
* FaceclawAppleMusic's MediaPlayer queries. Playing an item queues its
* collection on the system Music player.
*/
export class AppleMusicBrowserBridge implements MediaBrowserBridge {
private connected = false;
/** Bumped on every connect/disconnect; answers to older requests are dropped. */
private generation = 0;
listBrowsableApps(): MediaBrowserApp[] {
return FaceclawAppleMusic.shared().hasPermission() ? [APPLE_MUSIC_APP] : [];
}
connect(app: MediaBrowserApp): Promise<string> {
this.disconnect();
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS"));
if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed"));
this.connected = true;
return Promise.resolve("root");
}
browse(parentId: string): Promise<MediaBrowseItem[]> {
if (!this.connected) return Promise.reject(new Error("Not connected"));
const generation = this.generation;
return new Promise<MediaBrowseItem[]>((resolve, reject) => {
FaceclawAppleMusic.shared().browseCompletion(parentId, (json: string | null, error: string | null) => {
if (generation !== this.generation) reject(new Error("Disconnected"));
else if (error || json == null) reject(new Error(String(error || "Browse failed")));
else {
try {
resolve(parseMediaBrowseItems(String(json)));
} catch (parseError) {
reject(new Error(`Browse result parse failed: ${parseError}`));
}
}
});
});
}
playFromMediaId(mediaId: string): void {
if (!this.connected) return;
FaceclawAppleMusic.shared().playCompletion(mediaId, (error: string | null) => {
if (error) console.warn(`[apple-music] play ${mediaId} failed: ${error}`);
});
}
disconnect(): void {
this.connected = false;
this.generation++;
}
}
export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge();
+5 -31
View File
@@ -1,22 +1,11 @@
import { Utils } from "@nativescript/core";
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
import { onSettingsStoreChanged } from "./settings-store";
import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types";
declare const com: any;
export type MediaBrowserApp = {
packageName: string;
serviceClass: string;
appName: string;
};
export type MediaBrowseItem = {
mediaId: string;
title: string;
subtitle: string;
browsable: boolean;
playable: boolean;
};
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
type PendingRequest<T> = {
resolve: (value: T) => void;
@@ -29,7 +18,7 @@ type PendingRequest<T> = {
* start playback, including starting a player that is not currently running.
* Holds at most one connection at a time.
*/
export class FaceclawMediaBrowserBridge {
export class FaceclawMediaBrowserBridge implements MediaBrowserBridge {
private browser: any = null;
private listenerProxy: any = null;
private nextRequestId = 1;
@@ -155,22 +144,7 @@ export class FaceclawMediaBrowserBridge {
return;
}
try {
const raw = JSON.parse(json) as Array<{
mediaId?: string;
title?: string;
subtitle?: string;
browsable?: boolean;
playable?: boolean;
}>;
pending.resolve(
raw.map((item) => ({
mediaId: String(item.mediaId ?? ""),
title: String(item.title ?? ""),
subtitle: String(item.subtitle ?? ""),
browsable: Boolean(item.browsable),
playable: Boolean(item.playable),
})),
);
pending.resolve(parseMediaBrowseItems(json));
} catch (parseError) {
pending.reject(new Error(`Browse result parse failed: ${parseError}`));
}
@@ -183,4 +157,4 @@ export class FaceclawMediaBrowserBridge {
}
}
export const mediaBrowserBridge = new FaceclawMediaBrowserBridge();
export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge();
+231
View File
@@ -0,0 +1,231 @@
import { Dialogs, Utils } from "@nativescript/core";
import type { GrayImage } from "../graphics/image";
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
import { fromData } from "./kotlin-data";
import {
APPLE_MUSIC_APP_NAME,
APPLE_MUSIC_PACKAGE,
DEFAULT_MEDIA_STATE,
type MediaControllerBridge,
type MediaControllerState,
type MediaPlaybackState,
type MediaQueueItem,
} from "./media-types";
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
declare const FaceclawAppleMusic: any;
declare const FaceclawKitIosProtocol: any;
/**
* Re-read the system player this often while something listens: its change
* notifications are not delivered while Faceclaw is suspended, so a track
* change in the background would otherwise wait for the next one.
*/
const POLL_MS = 2_000;
/** Normalize FaceclawAppleMusic.stateJson()'s object. */
export function appleMusicState(raw: any): MediaControllerState {
if (!raw || raw.access !== true) {
return { ...DEFAULT_MEDIA_STATE, status: "Apple Music access is required." };
}
const hasItem = Boolean(raw.hasItem);
const playbackState: MediaPlaybackState = !hasItem
? "idle"
: raw.state === "playing" || raw.state === "paused" || raw.state === "stopped" ? raw.state : "unknown";
const rate = Number(raw.rate);
return {
available: hasItem,
accessEnabled: true,
packageName: APPLE_MUSIC_PACKAGE,
appName: APPLE_MUSIC_APP_NAME,
playbackState,
title: String(raw.title ?? ""),
artist: String(raw.artist ?? ""),
album: String(raw.album ?? ""),
positionMs: finiteOr(raw.positionMs, -1),
durationMs: finiteOr(raw.durationMs, -1),
// The system player reports a zero rate while it is still starting up.
playbackSpeed: playbackState === "playing" ? (rate > 0 ? rate : 1) : 0,
status: hasItem ? "" : "Nothing is playing in Apple Music.",
canPlayPause: hasItem,
canSkipNext: hasItem,
canSkipPrevious: hasItem,
artVersion: finiteOr(raw.artVersion, 0),
};
}
function finiteOr(value: unknown, fallback: number): number {
const number = Number(value);
return typeof value === "number" && Number.isFinite(number) ? number : fallback;
}
/** Everything a listener would redraw for; the position advances on its own. */
function stateSignature(state: MediaControllerState): string {
return JSON.stringify({ ...state, positionMs: 0 });
}
/**
* Apple Music through FaceclawAppleMusic (MediaPlayer's system player): iOS
* gives other apps' players no shared session API, so the system Music app is
* the only player Faceclaw can follow and control. Snapshots read the player
* directly (cheap property reads), so they are current even when a change
* notification was missed.
*/
export class AppleMusicControllerBridge implements MediaControllerBridge {
readonly accessPrompt = "Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone.";
readonly accessLabel = "Apple Music access needed";
readonly accessAction = "allow access";
readonly skipRebuildsQueue = false;
private readonly stateListeners = new Set<(state: MediaControllerState) => void>();
private bound = false;
private started = false;
private pollTimer: ReturnType<typeof setInterval> | null = null;
private lastSignature = "";
private pendingAccess: Promise<boolean> | null = null;
private protocol: any = null;
onStateChange(listener: (state: MediaControllerState) => void): () => void {
void this.start();
this.stateListeners.add(listener);
this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS);
listener(this.snapshot());
return () => {
this.stateListeners.delete(listener);
if (!this.stateListeners.size && this.pollTimer !== null) {
clearInterval(this.pollTimer);
this.pollTimer = null;
}
};
}
snapshot(): MediaControllerState {
let raw: unknown = null;
try {
raw = JSON.parse(String(this.native().stateJson()));
} catch (error) {
console.warn(`[apple-music] state read failed: ${error}`);
}
return appleMusicState(raw);
}
async start(): Promise<void> {
if (this.started) return;
this.started = true;
this.native().start();
}
async stop(): Promise<void> {
if (!this.started) return;
this.started = false;
this.native().stop();
}
async playPause(): Promise<void> {
this.native().playPause();
}
async skipNext(): Promise<void> {
this.native().skipNext();
}
async skipPrevious(): Promise<void> {
this.native().skipPrevious();
}
hasAccess(): boolean {
return Boolean(this.native().hasPermission());
}
/** Concurrent requests (glasses tap, phone Permissions card) share one prompt. */
requestAccess(): Promise<boolean> {
if (this.hasAccess()) return Promise.resolve(true);
this.pendingAccess ??= this.askForAccess().finally(() => {
this.pendingAccess = null;
this.refresh();
});
return this.pendingAccess;
}
async skipToQueueItem(id: number): Promise<void> {
this.native().skipToQueueIndex(Math.round(id));
}
getMediaVolumePercent(): number {
const volume = Number(this.native().volumePercent());
return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1;
}
setMediaVolumePercent(volumePercent: number): void {
this.native().setVolumePercent(Math.max(0, Math.min(100, Math.round(volumePercent))));
}
getAlbumArt(maxSize: number): GrayImage | null {
const packet = this.native().albumArt(Math.round(maxSize));
if (!packet) return null;
// The shared Kotlin tone curve and dither, as on Android.
this.protocol ??= FaceclawKitIosProtocol.new();
return grayImageFromPacket(fromData(this.protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither)));
}
getQueue(): MediaQueueItem[] {
try {
const raw = JSON.parse(String(this.native().queueJson()));
if (!Array.isArray(raw)) return [];
return raw.map((item: { id?: number; title?: string; active?: boolean }) => ({
id: Number(item?.id ?? -1),
title: String(item?.title ?? ""),
active: Boolean(item?.active),
}));
} catch (error) {
console.warn(`[apple-music] queue parse failed: ${error}`);
return [];
}
}
private native(): any {
const music = FaceclawAppleMusic.shared();
if (!this.bound) {
this.bound = true;
music.changeHandler = () => this.refresh();
}
return music;
}
private refresh(): void {
if (!this.stateListeners.size) return;
const state = this.snapshot();
const signature = stateSignature(state);
if (signature === this.lastSignature) return;
this.lastSignature = signature;
for (const listener of this.stateListeners) listener(state);
}
private async askForAccess(): Promise<boolean> {
const music = this.native();
const status = String(music.permissionStatus());
if (status === "denied") {
const open = await Dialogs.confirm({
title: "Apple Music Access",
message: "Allow Media & Apple Music access in Settings to show and control Apple Music on your glasses.",
okButtonText: "Open Settings",
cancelButtonText: "Cancel",
});
if (open) Utils.openUrl("app-settings:");
return false;
}
if (status === "restricted") {
await Dialogs.alert({
title: "Apple Music Access Restricted",
message: "Media library access is restricted on this iPhone. Check Screen Time or device management settings.",
okButtonText: "OK",
});
return false;
}
return new Promise<boolean>((resolve) => {
music.requestPermission((granted: boolean) => resolve(Boolean(granted)));
});
}
}
export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge();
+26 -59
View File
@@ -3,61 +3,25 @@ import { GrayImage } from "../graphics/image";
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
import { ignoredMediaPackages, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
import { onSettingsStoreChanged } from "./settings-store";
import { isNotificationListenerEnabled } from "./notification-access";
import {
DEFAULT_MEDIA_STATE,
interpolateMediaState,
type MediaControllerBridge,
type MediaControllerState,
type MediaPlaybackState,
type MediaQueueItem,
} from "./media-types";
declare const com: any;
export type MediaQueueItem = {
id: number;
title: string;
active: boolean;
};
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
export type MediaPlaybackState =
| "notification-access-required"
| "idle"
| "playing"
| "paused"
| "buffering"
| "stopped"
| "unknown";
export type MediaControllerState = {
available: boolean;
accessEnabled: boolean;
packageName: string;
appName: string;
playbackState: MediaPlaybackState;
title: string;
artist: string;
album: string;
positionMs: number;
durationMs: number;
playbackSpeed: number;
status: string;
canPlayPause: boolean;
canSkipNext: boolean;
canSkipPrevious: boolean;
};
const DEFAULT_MEDIA_STATE: MediaControllerState = {
available: false,
accessEnabled: false,
packageName: "",
appName: "",
playbackState: "idle",
title: "",
artist: "",
album: "",
positionMs: -1,
durationMs: -1,
playbackSpeed: 0,
status: "Media controller unavailable.",
canPlayPause: false,
canSkipNext: false,
canSkipPrevious: false,
};
export class FaceclawMediaControllerBridge {
export class FaceclawMediaControllerBridge implements MediaControllerBridge {
readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings.";
readonly accessLabel = "Notification access needed";
readonly accessAction = "open settings";
readonly skipRebuildsQueue = true;
private readonly stateListeners = new Set<(state: MediaControllerState) => void>();
private controller: any = null;
private listenerProxy: any = null;
@@ -72,12 +36,7 @@ export class FaceclawMediaControllerBridge {
}
snapshot(): MediaControllerState {
const state = { ...this.state };
if (state.playbackState === "playing" && state.positionMs >= 0 && state.playbackSpeed !== 0) {
state.positionMs += (Date.now() - this.stateUpdatedAtMs) * state.playbackSpeed;
state.positionMs = Math.max(0, state.durationMs > 0 ? Math.min(state.positionMs, state.durationMs) : state.positionMs);
}
return state;
return interpolateMediaState(this.state, this.stateUpdatedAtMs, Date.now());
}
async start(): Promise<void> {
@@ -108,9 +67,15 @@ export class FaceclawMediaControllerBridge {
this.controller?.skipPrevious();
}
openNotificationAccessSettings(): void {
hasAccess(): boolean {
return isNotificationListenerEnabled();
}
/** Opens the notification-access settings screen; the grant is only visible once the app resumes. */
async requestAccess(): Promise<boolean> {
this.ensureController();
this.controller?.openNotificationAccessSettings();
return false;
}
/** Jump playback to a queue item (see getQueue). */
@@ -206,6 +171,7 @@ export class FaceclawMediaControllerBridge {
canSkipPrevious: boolean,
accessEnabled: boolean,
status: string,
artVersion: number,
) => {
this.state = {
available: Boolean(title || artist || album || packageName) || String(playbackState) !== "idle",
@@ -223,6 +189,7 @@ export class FaceclawMediaControllerBridge {
canPlayPause: Boolean(canPlayPause),
canSkipNext: Boolean(canSkipNext),
canSkipPrevious: Boolean(canSkipPrevious),
artVersion: Number(artVersion),
};
this.stateUpdatedAtMs = Date.now();
for (const listener of this.stateListeners) {
@@ -248,4 +215,4 @@ function normalizePlaybackState(value: string): MediaPlaybackState {
}
}
export const mediaControllerBridge = new FaceclawMediaControllerBridge();
export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge();
+161
View File
@@ -0,0 +1,161 @@
import type { GrayImage } from "../graphics/image";
/**
* Shapes shared by the media bridges: media-controller.ts / media-browser.ts
* drive Android media sessions and MediaBrowserServices; their .ios.ts
* counterparts drive Apple Music through the MediaPlayer framework.
*/
/** The player behind the iOS bridges: the system Music app. */
export const APPLE_MUSIC_APP_NAME = "Apple Music";
export const APPLE_MUSIC_PACKAGE = "com.apple.Music";
export type MediaQueueItem = {
id: number;
title: string;
active: boolean;
};
export type MediaPlaybackState =
| "notification-access-required"
| "idle"
| "playing"
| "paused"
| "buffering"
| "stopped"
| "unknown";
export type MediaControllerState = {
available: boolean;
accessEnabled: boolean;
packageName: string;
appName: string;
playbackState: MediaPlaybackState;
title: string;
artist: string;
album: string;
positionMs: number;
durationMs: number;
playbackSpeed: number;
status: string;
canPlayPause: boolean;
canSkipNext: boolean;
canSkipPrevious: boolean;
/**
* Changes whenever getAlbumArt's result may have changed (any metadata
* update or session switch). Key cached art on this, not the title:
* players re-publish the same strings with a different bitmap.
*/
artVersion: number;
};
export const DEFAULT_MEDIA_STATE: MediaControllerState = {
available: false,
accessEnabled: false,
packageName: "",
appName: "",
playbackState: "idle",
title: "",
artist: "",
album: "",
positionMs: -1,
durationMs: -1,
playbackSpeed: 0,
status: "Media controller unavailable.",
canPlayPause: false,
canSkipNext: false,
canSkipPrevious: false,
artVersion: 0,
};
/** Interpolate the position of a playing item from the last state update. */
export function interpolateMediaState(state: MediaControllerState, updatedAtMs: number, nowMs: number): MediaControllerState {
const result = { ...state };
if (result.playbackState === "playing" && result.positionMs >= 0 && result.playbackSpeed !== 0) {
result.positionMs += (nowMs - updatedAtMs) * result.playbackSpeed;
result.positionMs = Math.max(0, result.durationMs > 0 ? Math.min(result.positionMs, result.durationMs) : result.positionMs);
}
return result;
}
export interface MediaControllerBridge {
/** Explains, on the glasses, what access is missing and that a tap asks for it. */
readonly accessPrompt: string;
/** The missing access as a short card label. */
readonly accessLabel: string;
/** What clicking the access prompt does, for its gesture hint. */
readonly accessAction: string;
/**
* Whether jumping to a queue item rebuilds the queue with that item first
* (Android players do); otherwise the queue keeps its order.
*/
readonly skipRebuildsQueue: boolean;
onStateChange(listener: (state: MediaControllerState) => void): () => void;
snapshot(): MediaControllerState;
start(): Promise<void>;
stop(): Promise<void>;
playPause(): Promise<void>;
skipNext(): Promise<void>;
skipPrevious(): Promise<void>;
hasAccess(): boolean;
/** Show the grant UI: a system settings screen or a permission prompt on the phone. */
requestAccess(): Promise<boolean>;
/** Jump playback to a queue item (see getQueue). */
skipToQueueItem(id: number): Promise<void>;
/** Media volume normalized to 0..100; -1 when unavailable. */
getMediaVolumePercent(): number;
setMediaVolumePercent(volumePercent: number): void;
/**
* Album art for the current item, grayscale with the shared photo tone
* (gamma-corrected and dithered onto the display's 16 levels), scaled to
* fit maxSize; null when the player provides none.
*/
getAlbumArt(maxSize: number): GrayImage | null;
/** The player's queue (playlist), empty when the player exposes none. */
getQueue(): MediaQueueItem[];
}
export type MediaBrowserApp = {
packageName: string;
serviceClass: string;
appName: string;
};
export type MediaBrowseItem = {
mediaId: string;
title: string;
subtitle: string;
browsable: boolean;
playable: boolean;
};
export interface MediaBrowserBridge {
/** Apps whose library can be browsed; cached after the first call. */
listBrowsableApps(refresh?: boolean): MediaBrowserApp[];
/** Connect to an app's library, replacing any existing connection. Resolves to the browse-tree root id. */
connect(app: MediaBrowserApp): Promise<string>;
/** Fetch one level of the connected library's tree. */
browse(parentId: string): Promise<MediaBrowseItem[]>;
/** Start playback of a browsed item. */
playFromMediaId(mediaId: string): void;
/** Drop the connection; in-flight requests are rejected. */
disconnect(): void;
}
/** Parse a browse listing's JSON rows. */
export function parseMediaBrowseItems(json: string): MediaBrowseItem[] {
const raw = JSON.parse(json) as Array<{
mediaId?: string;
title?: string;
subtitle?: string;
browsable?: boolean;
playable?: boolean;
}>;
return raw.map((item) => ({
mediaId: String(item.mediaId ?? ""),
title: String(item.title ?? ""),
subtitle: String(item.subtitle ?? ""),
browsable: Boolean(item.browsable),
playable: Boolean(item.playable),
}));
}
+18 -4
View File
@@ -1,5 +1,6 @@
import { nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings";
import { fetchTextWithUserAgent } from "../util/http";
import { bolusKindFromTreatment, type BolusKind } from "../apps/nightscout/missed-bolus-detector";
export type NightscoutPoint = {
timestampMs: number;
@@ -14,6 +15,8 @@ export type NightscoutCarbEvent = {
export type NightscoutBolusEvent = {
timestampMs: number;
insulin: number;
/** Meal, manual, loop-automatic, or unmarked (see BolusKind). */
kind: BolusKind;
};
export type NightscoutBasalEvent = {
@@ -102,10 +105,17 @@ type NightscoutTreatmentResponse = {
absolute?: unknown;
rate?: unknown;
duration?: unknown;
automatic?: unknown;
isSMB?: unknown;
type?: unknown;
wizard?: unknown;
};
const NIGHTSCOUT_REFRESH_MS = 60_000;
const NIGHTSCOUT_HISTORY_COUNT = 30;
// Glucose history by time rather than count, so a 1-minute CGM still covers
// the graphs' 2 hours and the missed-bolus detector's 90-minute lookback.
const NIGHTSCOUT_HISTORY_WINDOW_MS = 3 * 60 * 60 * 1000;
const NIGHTSCOUT_HISTORY_MAX_COUNT = 200;
const NIGHTSCOUT_TREATMENT_LOOKBACK_MS = 3 * 60 * 60 * 1000;
// The v1 treatments API silently adds `created_at >= now - 4 days` to any
// query without an explicit date filter (lib/server/query.js enforceDateFilter),
@@ -208,13 +218,17 @@ export class NightscoutBridge {
this.emit();
return;
}
const historyStartMs = nowMs - NIGHTSCOUT_HISTORY_WINDOW_MS;
const treatmentStartIso = new Date(nowMs - NIGHTSCOUT_TREATMENT_LOOKBACK_MS).toISOString();
const siteChangeStartIso = new Date(nowMs - NIGHTSCOUT_SITE_CHANGE_LOOKBACK_MS).toISOString();
const tokenQuery = `token=${encodeURIComponent(apiToken)}`;
const [entries, deviceStatus, status, treatments, siteChanges] = await Promise.all([
fetchJson<NightscoutEntryResponse[]>(
`${siteUrl}/api/v1/entries.json?count=${NIGHTSCOUT_HISTORY_COUNT}&${tokenQuery}`,
),
`${siteUrl}/api/v1/entries.json?find%5Bdate%5D%5B%24gte%5D=${historyStartMs}&count=${NIGHTSCOUT_HISTORY_MAX_COUNT}&${tokenQuery}`,
).then(async (recent) => recent?.length
? recent
// Nothing in the window: still show the last reading, as stale.
: await fetchJson<NightscoutEntryResponse[]>(`${siteUrl}/api/v1/entries.json?count=1&${tokenQuery}`)),
fetchJson<NightscoutDeviceStatusResponse[]>(
`${siteUrl}/api/v1/devicestatus.json?count=1&${tokenQuery}`,
),
@@ -334,7 +348,7 @@ function parseTreatments(treatments: NightscoutTreatmentResponse[], nowMs: numbe
const insulinValue = firstFiniteNumber(treatment.insulin);
if (insulinValue !== null && insulinValue > 0) {
boluses.push({ timestampMs, insulin: insulinValue });
boluses.push({ timestampMs, insulin: insulinValue, kind: bolusKindFromTreatment(treatment) });
}
const eventType = typeof treatment.eventType === "string" ? treatment.eventType : "";
+3
View File
@@ -2,3 +2,6 @@ import { iosNotificationState, iosNotificationMessage } from './notification-ico
export function isNotificationListenerEnabled(): boolean { return iosNotificationState() === 'ready' }
export function requestNotificationListenerAccess(): void { /* iOS exposes no deep link to this Bluetooth setting. */ }
export function notificationEmptyMessage(): string { return iosNotificationMessage() }
export type NotificationListenerState = "connected" | "no-access" | "connecting" | "stuck"
export function notificationListenerState(): NotificationListenerState { return isNotificationListenerEnabled() ? "connected" : "no-access" }
export function watchNotificationListenerBinding(): void { /* ANCS has no listener binding to watch. */ }
+39 -2
View File
@@ -32,7 +32,44 @@ export function requestNotificationListenerAccess(): void {
com.faceclaw.app.FaceclawEvenAppDetector.openNotificationAccessSettings(context);
}
/**
* Whether Android has the listener connected, and if not, why not:
* "connecting" is access granted but not (yet, or again) bound; "stuck" is
* still unbound after the native watchdog's forced rebinds, which only the
* user toggling notification access is sure to fix.
*/
export type NotificationListenerState = "connected" | "no-access" | "connecting" | "stuck";
export function notificationListenerState(): NotificationListenerState {
if (!global.isAndroid) return "connected";
const context = Utils.android.getApplicationContext();
if (!context) return "no-access";
return String(
com.faceclaw.app.FaceclawMediaNotificationListenerService.getListenerState(context),
) as NotificationListenerState;
}
/**
* Start the native watchdog that rebinds the listener when Android drops it
* while access is granted (see FaceclawMediaNotificationListenerService
* .watchListenerBinding). Called once at boot.
*/
export function watchNotificationListenerBinding(): void {
if (!global.isAndroid) return;
const context = Utils.android.getApplicationContext();
if (!context) return;
com.faceclaw.app.FaceclawMediaNotificationListenerService.watchListenerBinding(context);
}
export function notificationEmptyMessage(): string {
return isNotificationListenerEnabled() ? "No current Android notifications."
: "Grant permission on your phone to view notifications on the glasses.";
switch (notificationListenerState()) {
case "no-access":
return "Grant permission on your phone to view notifications on the glasses.";
case "connecting":
return "Connecting to Android notifications...";
case "stuck":
return "Android stopped sending notifications to Faceclaw. Turn Faceclaw's notification access off and on again in phone settings.";
case "connected":
return "No current Android notifications.";
}
}
+28 -8
View File
@@ -1,8 +1,10 @@
import { GrayImage } from '../graphics/image'
import { renderIcon, type IconName } from '../graphics/icons'
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
import { rememberNotificationSources } from './notification-sources'
import type { AndroidNotification } from './notification-types'
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
export type { NotificationIconsResult, NotificationIconResult } from './notification-icons'
export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons'
export const ALL_NOTIFICATIONS = 0x7fffffff
let active: AncsClient | null = null
@@ -26,8 +28,21 @@ export function dismissNotification(key: string): boolean { return active?.dismi
export function onAndroidNotificationPosted(listener: (key: string) => void): () => void {
listeners.add(listener); return () => { listeners.delete(listener) }
}
// ANCS does not provide app icons. Use a local bell; never fetch notification
// sources or content from a third-party icon service.
// ANCS removals carry no key and already reach the posted listeners above
// (they hear every change), and an open popup closes on the repaint that follows.
export function onAndroidNotificationRemoved(_listener: (key: string) => void): () => void { return () => {} }
export function onAndroidNotificationsChanged(listener: () => void): () => void { return onAndroidNotificationPosted(listener) }
// ANCS does not provide app icons, so show one for the notification's ANCS
// category; never fetch notification sources or content from a third-party
// icon service. Indexed by CategoryID: Other, IncomingCall, MissedCall,
// Voicemail, Social, Schedule, Email, News, HealthAndFitness,
// BusinessAndFinance, Location, Entertainment.
const CATEGORY_ICONS: readonly IconName[] = ['bell','phone-incoming','phone-missed','voicemail','message-circle',
'calendar','mail','newspaper','heart-pulse','briefcase','map-pin','tv']
function categoryIcon(category: string): GrayImage {
return renderIcon(CATEGORY_ICONS[Number(category)] ?? 'bell',24)?.clone() ?? bell()
}
// Drawn fallback for when the SVG rasterizer fails.
function bell(): GrayImage {
const image = new GrayImage(24,24,0)
image.fillRoundedRect(6,5,12,13,220,5)
@@ -37,11 +52,16 @@ function bell(): GrayImage {
}
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
// One icon per source, standing for its first notification.
const sources = new Map<string, string>()
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n.key)
const keys = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
return {icons: keys.map(() => bell()),keys,stale:false}
const sources = new Map<string, AndroidNotification>()
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
return {icons: shown.map(n => categoryIcon(n.category)),keys: shown.map(n => n.key),stale:false}
}
export function readNotificationIconByKey(key: string, _allowStale: boolean) {
return {icon: readActiveNotifications(ALL_NOTIFICATIONS).some(n => n.key === key) ? bell() : null,stale:false}
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key)
return {icon: notification ? categoryIcon(notification.category) : null,stale:false}
}
// ANCS carries no attachments.
export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
return {picture: null,stale:false}
}
+166 -20
View File
@@ -2,6 +2,7 @@ import { GrayImage } from "../graphics/image";
import { logCurrent, spanCurrent } from "./frame-timings";
import { toUint8Array } from "../util/array-util";
import { rememberNotificationSources } from "./notification-sources";
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
declare const com: any;
declare const java: any;
@@ -10,7 +11,7 @@ const ICON_SIZE = 24;
/** Ask the native listener for every active source, including those below the list's display limit. */
export const ALL_NOTIFICATIONS = 0x7fffffff;
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
// notification is posted or dismissed (see invalidateIconCaches callers), so
// notification is posted or removed (see invalidateIconCaches callers), so
// the tray is kept fresh by invalidation, not by expiry. A short TTL just
// forced a blocking main-thread re-fetch (~40-100ms of Java icon
// rasterization) every few seconds — usually producing a no-change frame that
@@ -22,8 +23,16 @@ let cachedKeys: string[] = [];
let cachedAtMs = 0;
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
const KEYED_ICON_CACHE_MAX = 128;
/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */
const pictureCache = new Map<string, {
postTime: number; maxWidth: number; maxHeight: number; picture: GrayImage | null; atMs: number;
}>();
const PICTURE_CACHE_MAX = 8;
let notificationListenerProxy: any = null;
const notificationPostedListeners = new Set<(notificationKey: string) => void>();
const notificationRemovedListeners = new Set<(notificationKey: string) => void>();
let listenerStateObserver: any = null;
const listenerStateListeners = new Set<() => void>();
function invalidateIconCaches(): void {
cachedAtMs = 0;
@@ -58,19 +67,21 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
const now = Date.now();
if (cachedAtMs > 0 && now - cachedAtMs < ICON_CACHE_MS) {
logCurrent("notification icons served from cache");
return { icons: cachedIcons.map(icon => icon.clone()), keys: cachedKeys.slice(), stale: false };
return cachedIconsResult(maxIcons, false);
}
if (allowStale) {
logCurrent("notification icons served stale");
return { icons: cachedIcons.map(icon => icon.clone()), keys: cachedKeys.slice(), stale: true };
return cachedIconsResult(maxIcons, true);
}
// Every icon, whatever this caller wants: the cache serves callers asking
// for different counts (the switcher row needs them all to lay itself out).
const keyList = new java.util.ArrayList();
const bytes = spanCurrent("fetch-notification-icons", () =>
toUint8Array(
com.faceclaw.app.FaceclawMediaNotificationListenerService.getActiveNotificationIconGrays(
ICON_SIZE,
maxIcons,
ALL_NOTIFICATIONS,
keyList,
),
),
@@ -91,7 +102,12 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea
cachedIcons = icons;
cachedKeys = keys;
cachedAtMs = now;
return { icons: icons.map(icon => icon.clone()), keys: keys.slice(), stale: false };
return cachedIconsResult(maxIcons, false);
}
/** Copies of the first maxIcons cached icons, with their keys. */
function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult {
return { icons: cachedIcons.slice(0, maxIcons).map(icon => icon.clone()), keys: cachedKeys.slice(0, maxIcons), stale };
}
export type NotificationIconResult = {
@@ -143,6 +159,71 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not
return { icon: icon?.clone() ?? null, stale: false };
}
export type NotificationPictureResult = {
/** Shared with the cache: blit it, don't draw on it. */
picture: GrayImage | null;
/** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */
stale: boolean;
};
/**
* The picture attached to a notification (hasPicture), scaled to fit
* maxWidth x maxHeight with the photo tone and inverted if it is drawn on a
* light background (GrayPacket.hasLightBackground), under the allow-stale contract of
* readNotificationIconByKey. Cached per post of the notification: an ongoing
* notification updated in place (Dexcom redraws its graph every few minutes)
* fires no posted event, but its postTime moves. ICON_CACHE_MS is the
* backstop here too. Under allowStale an earlier fetch stands in, if it fits.
*/
export function readNotificationPicture(
notification: AndroidNotification,
maxWidth: number,
maxHeight: number,
allowStale: boolean,
): NotificationPictureResult {
const width = Math.round(maxWidth);
const height = Math.round(maxHeight);
if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) {
return { picture: null, stale: false };
}
const now = Date.now();
const cached = pictureCache.get(notification.key);
if (
cached && cached.postTime === notification.postTime && cached.maxWidth === width
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
) {
return { picture: cached.picture, stale: false };
}
if (allowStale) {
const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height;
return { picture: fits ? cached!.picture : null, stale: true };
}
const picture = spanCurrent("fetch-notification-picture", () =>
grayImageFromPacket(
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray(
notification.key,
width,
height,
PHOTO_TONE.gamma,
PHOTO_TONE.dither,
// A chart or screenshot made for a light theme would be a glaring
// lit panel on the glasses: show it light-on-dark.
true,
),
),
);
pictureCache.delete(notification.key);
pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now });
while (pictureCache.size > PICTURE_CACHE_MAX) {
const oldestKey = pictureCache.keys().next().value;
if (oldestKey === undefined) break;
pictureCache.delete(oldestKey);
}
return { picture, stale: false };
}
export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] {
if (!global.isAndroid || maxNotifications <= 0) return [];
try {
@@ -183,12 +264,68 @@ export function dismissNotification(notificationKey: string): boolean {
}
export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void {
notificationPostedListeners.add(listener);
ensureNotificationPostedListener();
return addNotificationListener(notificationPostedListeners, listener);
}
/**
* A listed notification went away: cancelled by its source (including one
* withdrawn moments after it was posted), dismissed on the phone or from the
* glasses, or updated into one the glasses don't list.
*/
export function onAndroidNotificationRemoved(listener: (notificationKey: string) => void): () => void {
return addNotificationListener(notificationRemovedListeners, listener);
}
/**
* Any change to the listed notifications (a post, a removal, or the listener
* connecting or going missing), for views that just repaint.
*/
export function onAndroidNotificationsChanged(listener: () => void): () => void {
const offPosted = onAndroidNotificationPosted(listener);
const offRemoved = onAndroidNotificationRemoved(listener);
const offState = onListenerStateChanged(listener);
return () => {
notificationPostedListeners.delete(listener);
if (notificationPostedListeners.size === 0) {
removeNotificationPostedListener();
offPosted();
offRemoved();
offState();
};
}
function onListenerStateChanged(listener: () => void): () => void {
listenerStateListeners.add(listener);
if (global.isAndroid && !listenerStateObserver) {
listenerStateObserver = new java.lang.Runnable({
run: () => {
invalidateIconCaches();
const snapshot = Array.from(listenerStateListeners);
setTimeout(() => {
for (const each of snapshot) {
each();
}
}, 0);
},
});
com.faceclaw.app.FaceclawMediaNotificationListenerService.addListenerStateObserver(listenerStateObserver);
}
return () => {
listenerStateListeners.delete(listener);
if (listenerStateListeners.size === 0 && listenerStateObserver) {
com.faceclaw.app.FaceclawMediaNotificationListenerService.removeListenerStateObserver(listenerStateObserver);
listenerStateObserver = null;
}
};
}
function addNotificationListener(
listeners: Set<(notificationKey: string) => void>,
listener: (notificationKey: string) => void,
): () => void {
listeners.add(listener);
ensureNativeNotificationListener();
return () => {
listeners.delete(listener);
if (notificationPostedListeners.size === 0 && notificationRemovedListeners.size === 0) {
removeNativeNotificationListener();
}
};
}
@@ -215,21 +352,19 @@ function normalizeNotification(value: any): AndroidNotification | null {
postTime: Number(value.postTime) || 0,
when: Number(value.when) || 0,
actions,
foregroundService: Boolean(value.foregroundService),
hasPicture: Boolean(value.hasPicture),
};
}
function ensureNotificationPostedListener(): void {
function ensureNativeNotificationListener(): void {
if (!global.isAndroid || notificationListenerProxy) return;
notificationListenerProxy = new com.faceclaw.app.FaceclawNotificationListener({
onNotificationPosted: (notificationKey: string) => {
invalidateIconCaches();
const key = String(notificationKey);
const listeners = Array.from(notificationPostedListeners);
setTimeout(() => {
for (const listener of listeners) {
listener(key);
}
}, 0);
dispatchNotificationEvent(notificationPostedListeners, notificationKey);
},
onNotificationRemoved: (notificationKey: string) => {
dispatchNotificationEvent(notificationRemovedListeners, notificationKey);
},
});
com.faceclaw.app.FaceclawMediaNotificationListenerService.addNotificationListener(
@@ -237,7 +372,18 @@ function ensureNotificationPostedListener(): void {
);
}
function removeNotificationPostedListener(): void {
function dispatchNotificationEvent(listeners: Set<(notificationKey: string) => void>, notificationKey: string): void {
invalidateIconCaches();
const key = String(notificationKey);
const snapshot = Array.from(listeners);
setTimeout(() => {
for (const listener of snapshot) {
listener(key);
}
}, 0);
}
function removeNativeNotificationListener(): void {
if (!global.isAndroid || !notificationListenerProxy) return;
com.faceclaw.app.FaceclawMediaNotificationListenerService.removeNotificationListener(
notificationListenerProxy,
+4
View File
@@ -5,6 +5,10 @@ export type AndroidNotification = {
bigText: string; subText: string; infoText: string; summaryText: string;
category: string; lines: string[]; postTime: number; when: number;
actions: AndroidNotificationAction[]; dismissLabel?: string;
/** Posted by a foreground service (Android): a "this app is working" status notification. */
foregroundService?: boolean;
/** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */
hasPicture?: boolean;
/** Full message size reported by iOS, even when the fetched text is bounded. */
messageSize?: number;
};
+5 -13
View File
@@ -1,6 +1,5 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
declare const com: any;
import { openSocket, type SocketConnection } from "./socket";
/**
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
@@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000;
export type OpenAiSttOptions = CloudSttOptions;
export class OpenAiRealtimeSttClient implements CloudSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// Base64 audio (or the commit sentinel) queued until the socket opens.
@@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
start(): void {
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
this.open = true;
@@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
},
});
};
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(
WS_URL,
this.listenerProxy,
"Authorization",
`Bearer ${this.options.apiKey}`,
);
this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` });
this.options.onStatus("Connecting to OpenAI...");
} catch (error) {
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
+16 -2
View File
@@ -85,12 +85,25 @@ export class PreviewDisplayTarget implements DisplayTarget {
this.composited();
}
/**
* As on the live bridge (FaceclawCommunicatorBridge.hasNativeSurface): a call that lands
* after its window closed, or on a target that replaced the one it was configured on, is
* dropped rather than failing the caller.
*/
private hasSurface(id: string, what: string): boolean {
if (this.compositor.hasSurface(id)) return true;
console.log(`[ios-preview] ${what} for unknown surface ${id} dropped`);
return false;
}
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
if (!this.hasSurface(id, "setSurfaceVisible")) return;
this.compositor.setSurfaceVisible(id, Boolean(visible));
this.composited();
}
async setSurfaceDepth(id: string, depth: number): Promise<void> {
if (!this.hasSurface(id, "setSurfaceDepth")) return;
this.compositor.setSurfaceDepth(id, Math.round(depth));
}
@@ -113,6 +126,7 @@ export class PreviewDisplayTarget implements DisplayTarget {
_frameId = 0,
glyphs: ArrayBuffer | null = null,
): Promise<void> {
if (!this.hasSurface(surfaceId, "frame")) return;
this.compositor.submitSurfaceFrame(surfaceId, pixels8bpp,
{ x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, glyphs);
this.composited();
@@ -138,8 +152,8 @@ export class PreviewDisplayTarget implements DisplayTarget {
return this.compositor.composite();
}
getCompositePreview(green = false): ImageSource | null {
return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green);
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green, crop);
}
/** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */
+11 -2
View File
@@ -160,9 +160,18 @@ export class PreviewDisplayTarget implements DisplayTarget {
return Promise.resolve("composited");
}
getCompositePreview(green = false): ImageSource | null {
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
if (!global.isAndroid) return null;
const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
const bitmap = crop
? this.compositor.getCompositePreviewBitmap(
PREVIEW_BRIGHTEN_GAMMA,
green,
Math.round(crop.x),
Math.round(crop.y),
Math.round(crop.width),
Math.round(crop.height),
)
: this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
return bitmap ? new ImageSource(bitmap) : null;
}
+3 -1
View File
@@ -24,6 +24,8 @@ export class ReconnectingSttClient implements CloudSttClient {
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
/** How long finish() waits for the provider's final transcript. */
readonly finalTimeoutMs = 5000,
) {}
start(): void {
@@ -90,7 +92,7 @@ export class ReconnectingSttClient implements CloudSttClient {
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), 5000);
this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs);
if (this.ready) this.client?.finish();
}
+21
View File
@@ -0,0 +1,21 @@
import { type Place } from './location-types'
declare const CLGeocoder: any, CLLocation: any
/** Geocoders with a lookup in flight, held until their callback so collection can't drop it. */
const pending = new Set<any>()
/** The place at a coordinate from Apple's geocoder, or null when it can't name it. */
export function reverseGeocode(latitude: number, longitude: number): Promise<Place | null> {
return new Promise(resolve => {
const geocoder = CLGeocoder.new()
pending.add(geocoder)
const location = CLLocation.alloc().initWithLatitudeLongitude(latitude, longitude)
geocoder.reverseGeocodeLocationCompletionHandler(location, (placemarks: any, error: any) => {
pending.delete(geocoder)
const placemark = !error && placemarks && placemarks.count ? placemarks.objectAtIndex(0) : null
resolve(placemark
? { locality: placemark.locality ?? '', region: placemark.administrativeArea ?? '', country: placemark.country ?? '' }
: null)
})
})
}
+20
View File
@@ -0,0 +1,20 @@
import { Utils } from "@nativescript/core";
import { type Place } from "./location-types";
declare const com: any;
/** The place at a coordinate from Android's Geocoder, or null when the phone can't name it. */
export function reverseGeocode(latitude: number, longitude: number): Promise<Place | null> {
const context = Utils.android.getApplicationContext();
if (!context) return Promise.resolve(null);
return new Promise<Place | null>((resolve) => {
const geocoder = new com.faceclaw.app.FaceclawReverseGeocoder(context);
// The lookup thread holds the listener proxy until its one callback.
geocoder.lookup(latitude, longitude, new com.faceclaw.app.FaceclawReverseGeocoder.Listener({
onResult: (locality: string | null, region: string | null, country: string | null) => {
resolve({ locality: locality ? String(locality) : "", region: region ? String(region) : "", country: country ? String(country) : "" });
},
}));
});
}
+14 -4
View File
@@ -1,7 +1,7 @@
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
import type { SocketConnection, SocketHeader, SocketListener } from './socket'
declare const FaceclawSocket: any
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
const native = FaceclawSocket.alloc().initWithURL(url)
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null)
let ended = false
const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() }
const timer = setInterval(() => {
@@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te
else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) }
}
}, 25)
return { sendText: text => { if (!ended) native.sendText(text) }, close }
// NSURLSession queues every send; a failed one surfaces as an error event.
return {
sendText: text => { if (!ended) native.sendText(text); return !ended },
sendBinary: bytes => {
if (ended) return false
const copy = bytes.slice()
native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
return true
},
close,
}
}
+14 -3
View File
@@ -1,9 +1,20 @@
import { toJavaBytes } from './cloud-stt'
declare const com: any
export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void }
export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } {
/** One extra handshake header, e.g. an API key. */
export type SocketHeader = { name: string; value: string }
export type SocketConnection = {
/** False when the frame was refused outright; later failures arrive via onFailure. */
sendText(text: string): boolean
sendBinary(bytes: Uint8Array): boolean
close(code: number, reason: string): void
}
export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection {
const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener)
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null)
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text),
const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null)
const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false,
sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false,
close: (code: number, reason: string) => native.close(code, reason) }
return connection
}
+7 -10
View File
@@ -1,8 +1,7 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt";
import { openSocket, type SocketConnection } from "./socket";
import { TimedTranscript } from "./transcript-format";
declare const com: any;
/**
* Soniox realtime speech-to-text over WebSocket.
* https://soniox.com/docs/stt/rt/real-time-transcription
@@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5";
export type SonioxSttOptions = CloudSttOptions;
export class SonioxSttClient implements CloudSttClient {
private ws: any = null;
private listenerProxy: any = null;
private ws: SocketConnection | null = null;
private open = false;
private closed = false;
// PCM queued until the socket opens and the config message is sent.
@@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient {
start(): void {
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
const listener = {
onOpen: () => {
if (this.closed) return;
// The API key rides in the config message; there is no auth header.
@@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient {
if (this.closed) return;
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
},
});
};
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.ws = openSocket(WS_URL, listener);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
@@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient {
}
this.ws = null;
}
this.listenerProxy = null;
this.pendingPcm.length = 0;
}
private sendPcm(pcm: Uint8Array): void {
try {
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
if (this.ws?.sendBinary(pcm) === false) {
this.options.onDisconnected?.("Soniox audio send failed.");
}
} catch (error) {
+156 -20
View File
@@ -1,12 +1,23 @@
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { WavRecorder } from './wav-recorder.ios'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise<void>
}
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
const CLOUD_FINAL_TIMEOUT_MS = 5000
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode.
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments. When the
* session or the app lifecycle cuts one short it is drained, not discarded;
* restarting it is the controller's call.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -24,6 +35,22 @@ export class IosVoiceControlBridge {
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: ReconnectingSttClient | null = null
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
// joins it instead of restarting the mic (transcripts reach every listener).
private continuous = false
// Shown instead of the native "ended" status once a drain the bridge started completes.
private drainStatus: string | null = null
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
@@ -38,16 +65,24 @@ export class IosVoiceControlBridge {
if (!this.native) {
this.native = FaceclawSpeech.new()
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
this.native.pcmHandler = (pcm: NSData) => {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
this.cloud?.acceptPcm(bytes)
if (this.continuous) this.record(bytes)
}
}
return this.native
}
async prepare(foreground: boolean, usePhoneMic = false): Promise<boolean> {
let status = FaceclawSpeech.authorizationStatus()
if (status === 0) {
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise<boolean> {
// Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio.
if (!usesCloudStt(voice)) {
let status = FaceclawSpeech.authorizationStatus()
if (status === 0) {
if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false }
status = await new Promise<number>(resolve => FaceclawSpeech.requestAuthorization(resolve))
}
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
}
if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false }
if (usePhoneMic) {
let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus()
if (microphoneStatus === 0) {
@@ -59,12 +94,14 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
const generation = ++this.generation
this.continuous = continuous
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '')
const startError = this.startNative(voice, endpointing, false)
if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return }
try {
await session.setMicrophone(true, packet => {
@@ -74,7 +111,7 @@ export class IosVoiceControlBridge {
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log('Voice: glasses microphone enabled; using on-device recognition')
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
@@ -86,35 +123,103 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise<void> {
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
this.continuous = continuous
this.log = log; this.source = 'phone'; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '')
const error = this.startNative(voice, endpointing, true)
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
log('Voice: phone default microphone enabled; using on-device recognition')
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
}
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
const native = this.ensureNative()
const generation = this.generation
const current = () => generation === this.generation
const cloud = createCloudSttClient(voice, {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => this.continuous)
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
this.cloud = cloud
cloud.start()
return ''
}
stopPhoneCapture(): void {
if (this.source !== 'phone') return
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* until called with another name or ''. Only cloud captures supply PCM.
*/
recordTo(name: string): void {
if (name === this.recordingName) return
this.recorder?.close(); this.recorder = null
this.recordingName = name
}
private record(pcm: Uint8Array): void {
if (!this.recorder && this.recordingName) {
this.recorder = WavRecorder.open(this.recordingName)
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
// One attempt per recording, not one per packet.
if (!this.recorder) this.recordingName = ''
}
this.recorder?.append(pcm)
}
/** End continuous capture, transcribing the audio since the last pause. */
stopContinuousCapture(): Promise<void> {
if (!this.continuous) return Promise.resolve()
return this.finalizeCapture()
}
/** Push-to-talk shares a listening continuous capture (not one that is only finalizing). */
private joinsContinuous(continuous: boolean): boolean {
if (continuous || !this.continuous || !this.capturing) return false
this.log('Voice: push-to-talk joins the running continuous capture')
return true
}
stopPushToTalk(): Promise<void> {
// The continuous capture owns the mic and keeps it.
if (this.continuous) return Promise.resolve()
return this.finalizeCapture()
}
private finalizeCapture(status: string | null = null): Promise<void> {
const completion = this.completion
if (!this.capturing) return completion
this.capturing = false; this.finalizing = true
this.drainStatus = status
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus('Recognizing…'); this.native?.finish()
return completion
}
stop(): void {
++this.generation
this.capturing = this.finalizing = false
this.source = null
this.capturing = this.finalizing = this.continuous = false
this.source = null; this.drainStatus = null
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.clearCloud()
this.finishCompletion()
}
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
/**
* The glasses session went away (or the glasses locked). Push-to-talk ends
* with `message`; continuous capture transcribes what it already heard and
* shows `continuousMessage`.
*/
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.',
continuousMessage = 'Glasses disconnected. Transcription resumes when they reconnect.'): void {
if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
this.stop(); this.setStatus(message); this.emitEnd()
}
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
@@ -122,6 +227,34 @@ export class IosVoiceControlBridge {
const session = this.session; this.session = null
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
private clearCloud(): void {
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
this.cloudTimer = null
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
}
/** The native audio has ended; have the provider finalize what it heard. */
private finishCloud(status: string): void {
this.capturing = false; this.finalizing = true
this.cloudEndStatus = status
this.setStatus('Recognizing…')
// Armed first: finish() can deliver the final synchronously.
this.cloudTimer = setTimeout(() => this.completeCloud(), this.cloud?.finalTimeoutMs ?? CLOUD_FINAL_TIMEOUT_MS)
this.cloud?.finish()
}
private completeCloud(): void {
const status = this.cloudEndStatus ?? 'Ready to send'
this.clearCloud()
this.capturing = this.finalizing = this.continuous = false
this.finishCompletion()
this.setStatus(status); this.emitEnd()
}
private receiveCloudTranscript(event: CloudSttTranscriptEvent): void {
if (!this.capturing && !this.finalizing) return
for (const fn of [...this.transcripts]) fn(event)
// Push-to-talk commits once, so the first final after it is the last.
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private finishCompletion(): void {
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
@@ -134,15 +267,18 @@ export class IosVoiceControlBridge {
} else if (event.kind === 'status') this.setStatus(event.message ?? '')
else if (event.kind === 'audio') {
this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`)
if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
// A cloud provider's own status (connecting, listening, reconnecting) stays up.
if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`)
} else if (event.kind === 'finishing') {
void this.stopPushToTalk(); this.emitEnd()
} else if (event.kind === 'ended') {
this.source = null
this.capturing = this.finalizing = false
const status = this.drainStatus ?? event.message ?? 'Ready to send'
this.source = null; this.drainStatus = null
this.clearAudioTimer(); this.releaseMicrophone()
if (this.cloud) { this.finishCloud(status); return }
this.capturing = this.finalizing = false
this.finishCompletion()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
this.setStatus(status); this.emitEnd()
}
}
}
+74 -49
View File
@@ -1,11 +1,8 @@
import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
@@ -31,7 +28,9 @@ export type VoiceControlState = {
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
// renamed, since it's a persisted settings value on real installs already.
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP
// (whisper-server-stt.ts).
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
@@ -55,6 +54,10 @@ export type PushToTalkOptions = {
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
@@ -125,13 +128,20 @@ export class FaceclawVoiceControlBridge {
private captureEndpointing = false;
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
private cloudClient: CloudSttClient | null = null;
// Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the
// decode-only "cloud" mode and every decoded PCM frame is broadcast to these
// listeners. STT capture preempts it — a live assistant/transcribe session
// owns the single G2 mic and the raw tap is stopped for its duration.
// What the running STT stream transcribes with (transcriptionKey); a holder
// asking for something else restarts the stream rather than sharing it.
private streamProvider: string | null = null;
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
// controller runs in the decode-only "cloud" mode and every decoded PCM
// frame is broadcast to these listeners. STT capture preempts it — a live
// assistant/transcribe session owns the single G2 mic and the raw tap is
// stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private rawActive = false;
// Who wants the raw tap. It runs while any owner does, so one owner letting
// go (an EvenHub app backgrounding) doesn't silence another.
private readonly rawOwners = new Set<string>();
// Set when the current capture session runs "my voice only" verification;
// a non-wearer verdict arrives just before the final transcript and makes
// the bridge swallow it.
@@ -210,6 +220,17 @@ export class FaceclawVoiceControlBridge {
this.releaseCapture("continuous", false);
}
/**
* Restart the running continuous capture when it transcribes with another
* provider than `options` asks for (a window that needs a multilingual one
* opened or closed), keeping every holder on the new stream. A parked
* capture is left alone: it takes the options when it resumes.
*/
retargetContinuousCapture(options: PushToTalkOptions): void {
if (!this.captureHolders.has("continuous") || !this.micIsLive()) return;
this.acquireCapture("continuous", options);
}
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
onRawPcm(listener: RawPcmListener): () => void {
this.rawPcmListeners.add(listener);
@@ -223,11 +244,33 @@ export class FaceclawVoiceControlBridge {
}
/**
* Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns
* false) if an STT capture already owns the mic; the caller re-tries when its
* eligibility is re-evaluated. Idempotent while already running.
* Start the decode-only raw-PCM tap on behalf of `owner`. Fails (returns
* false) if an STT capture already owns the mic; the owner stays registered
* and re-tries later (the EvenHub router when eligibility is re-evaluated,
* the Microphones app from its capture check). Idempotent while already
* running.
*/
startRawCapture(options: { communicator: any }): boolean {
startRawCapture(options: { communicator: any; owner: string }): boolean {
if (!global.isAndroid) return false;
this.rawOwners.add(options.owner);
return this.startRawTap(options.communicator);
}
/** `owner` no longer wants the raw tap; it stops once no owner does. */
stopRawCapture(owner: string): void {
this.rawOwners.delete(owner);
if (this.rawOwners.size === 0) this.endRawTap();
}
/**
* Whether the raw tap is running and its glasses mic enable still holds
* (or is still coming up). False after an STT preemption or a session drop.
*/
isRawCaptureLive(): boolean {
return this.rawActive && this.micIsLive();
}
private startRawTap(communicator: any): boolean {
if (!global.isAndroid) return false;
if (this.rawActive && this.micIsLive()) return true;
// An STT holder owns the mic even while its stream waits on a new glasses
@@ -237,7 +280,7 @@ export class FaceclawVoiceControlBridge {
if (this.started || this.rawActive) this.teardownCapture();
this.suspendedRaw = false;
this.ensureController();
this.controller?.setCommunicator(options.communicator);
this.controller?.setCommunicator(communicator);
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
// preview-mode capture may have left on the shared controller.
this.controller?.setUsePhoneMic(false);
@@ -257,7 +300,7 @@ export class FaceclawVoiceControlBridge {
return true;
}
stopRawCapture(): void {
private endRawTap(): void {
this.suspendedRaw = false;
if (!this.rawActive) return;
this.rawActive = false;
@@ -269,7 +312,7 @@ export class FaceclawVoiceControlBridge {
if (!global.isAndroid) return;
// STT preempts the raw-PCM tap: a live transcription session owns the mic,
// and an EvenHub app gets no audio for its duration.
if (this.rawActive) this.stopRawCapture();
if (this.rawActive) this.endRawTap();
// Whether the mic is already running is a question about the stream, not
// about this set: a holder whose capture died with the glasses session
// (or whose mic enable was refused) is still in it. Trusting the set here
@@ -278,13 +321,15 @@ export class FaceclawVoiceControlBridge {
const micLive = this.micIsLive();
this.suspendedHolders.delete(holder);
this.captureHolders.add(holder);
if (micLive) {
if (micLive && this.streamProvider === transcriptionKey(options)) {
// Mic already running; the new holder just shares the existing stream
// (transcripts are already broadcast to its listeners).
return;
}
// Any leftover capture is dead; drop it before starting the new one.
// Any leftover capture is dead, or transcribes with another provider than
// this holder needs; drop it before starting the new one.
if (this.started) this.teardownCapture();
this.streamProvider = transcriptionKey(options);
this.ensureController();
this.controller?.setCommunicator(options.communicator);
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
@@ -341,38 +386,12 @@ export class FaceclawVoiceControlBridge {
* than failing the capture outright.
*/
private createCloudClient(options: PushToTalkOptions): CloudSttClient | null {
if (options.provider === "onboard" || options.provider === "onboard-whisper") return null;
const sttOptions = {
apiKey: "",
return createCloudSttClient(options, {
onTranscript: (event: CloudSttTranscriptEvent) =>
this.emitTranscript(event.text, event.isFinal, event),
onStatus: (status: string) => this.setStatus(status),
onError: (message: string) => this.setStatus(message),
};
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
if (options.provider === "elevenlabs") {
const apiKey = options.elevenLabsApiKey.trim();
if (!apiKey) {
this.setStatus("No ElevenLabs key set; using on-device voice.");
return null;
}
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
if (!apiKey) {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return reconnecting((config) => new SonioxSttClient(config), apiKey);
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
return null;
}
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
}, () => this.captureHolders.has("continuous"));
}
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
@@ -437,6 +456,11 @@ export class FaceclawVoiceControlBridge {
return this.suspendedHolders.size > 0 || this.suspendedRaw;
}
/** Whether `holder` holds the mic, including across a session outage. */
holdsCapture(holder: CaptureHolder): boolean {
return this.captureHolders.has(holder) || this.suspendedHolders.has(holder);
}
/** Whether anyone holds the mic, including across a session outage. */
isCaptureHeld(): boolean {
return this.captureHolders.size > 0 || this.suspendedHolders.size > 0;
@@ -457,8 +481,8 @@ export class FaceclawVoiceControlBridge {
for (const holder of holders) {
this.acquireCapture(holder, { ...options, endpointing: this.captureEndpointing });
}
if (resumeRaw && !holders.length) {
this.startRawCapture({ communicator: options.communicator });
if (resumeRaw && !holders.length && this.rawOwners.size > 0) {
this.startRawTap(options.communicator);
}
}
@@ -476,6 +500,7 @@ export class FaceclawVoiceControlBridge {
/** Stop the native capture and any cloud client; leaves holders alone. */
private teardownCapture(): void {
this.rawActive = false;
this.streamProvider = null;
if (global.isAndroid) {
this.controller?.stop();
}
+22
View File
@@ -0,0 +1,22 @@
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
export function wavHeader(dataBytes: number): Uint8Array {
const header = new Uint8Array(44);
const view = new DataView(header.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + dataBytes, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, 16000, true);
view.setUint32(28, 32000, true); // byte rate
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
ascii(36, "data");
view.setUint32(40, dataBytes, true);
return header;
}
+75
View File
@@ -0,0 +1,75 @@
import { knownFolders, path as paths } from '@nativescript/core'
import { wavHeader } from './wav-header'
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
const FLUSH_BYTES = 32000
/**
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private pending: Uint8Array[] = []
private pendingBytes = 0
private dataBytes = 0
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.wav`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
return new WavRecorder(handle, path)
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.handle || !pcm.length) return
this.pending.push(pcm.slice())
this.pendingBytes += pcm.length
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
}
close(): void {
this.flush()
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
this.handle = null
}
private flush(): void {
const handle = this.handle
if (!handle || !this.pendingBytes) return
const chunk = new Uint8Array(this.pendingBytes)
let offset = 0
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
this.pending = []
this.pendingBytes = 0
try {
// NativeScript throws a failed call's NSError (storage full, file removed).
handle.writeDataError(toData(chunk))
handle.seekToOffsetError(0)
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
handle.seekToEndReturningOffsetError(null)
this.dataBytes += chunk.length
return
} catch {
// Stop recording rather than retry.
}
try { handle.closeAndReturnError() } catch { /* ignore */ }
this.handle = null
}
}
function toData(bytes: Uint8Array): NSData {
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
}
+296 -17
View File
@@ -1,11 +1,22 @@
import { hasLocationPermission } from "./location-permissions";
import { getCurrentLocation, type CurrentLocation } from "./location";
import { type Place } from "./location-types";
import { reverseGeocode } from "./reverse-geocode";
import { getStringSetting, setStringSetting } from "./settings-store";
import { fetchWithUserAgent } from "../util/http";
import { USER_AGENT } from "../version";
export type WeatherPhase = "permission-required" | "locating" | "loading" | "ready" | "error";
/** National Weather Service in the US, Open-Meteo everywhere else. */
export type WeatherSource = "nws" | "open-meteo";
/** Short source names for the Weather app's credit line. */
export const WEATHER_SOURCE_NAMES: Record<WeatherSource, string> = {
nws: "NWS",
"open-meteo": "Open-Meteo",
};
/** Sky and precipitation state, as coarse as the icons that show it. */
export type WeatherCondition =
| "clear"
@@ -48,6 +59,8 @@ export type ForecastPeriod = {
export type WeatherState = {
phase: WeatherPhase;
status: string;
/** Where current and forecast came from; null until the first refresh. */
source: WeatherSource | null;
locationName: string;
current: CurrentWeather | null;
forecast: ForecastPeriod[];
@@ -106,12 +119,56 @@ type NwsMeasure = {
unitCode?: unknown;
};
type OpenMeteoResponse = {
utc_offset_seconds?: unknown;
current?: {
time?: unknown;
temperature_2m?: unknown;
relative_humidity_2m?: unknown;
is_day?: unknown;
weather_code?: unknown;
wind_speed_10m?: unknown;
wind_direction_10m?: unknown;
};
hourly?: {
time?: unknown;
temperature_2m?: unknown;
precipitation_probability?: unknown;
weather_code?: unknown;
wind_speed_10m?: unknown;
wind_direction_10m?: unknown;
};
};
type LoadedWeather = {
source: WeatherSource;
locationName: string;
current: CurrentWeather;
forecast: ForecastPeriod[];
};
const NWS_API_ROOT = "https://api.weather.gov";
const NWS_HEADERS = {
Accept: "application/geo+json",
// NWS asks callers to identify themselves with a contact address.
"User-Agent": `${USER_AGENT} (https://github.com/jimrandomh/faceclaw)`,
};
// Free for non-commercial use without a key; data is CC BY 4.0, credited
// on the Weather app's source line and in ACKNOWLEDGEMENTS.md.
const OPEN_METEO_FORECAST_URL = "https://api.open-meteo.com/v1/forecast";
/**
* Days of hourly forecast to ask Open-Meteo for, from local midnight today:
* enough that MAX_FORECAST_PERIODS periods from any hour end before it does.
*/
const OPEN_METEO_FORECAST_DAYS = 8;
/** Local hours Open-Meteo forecast days and nights start at, as NWS periods do. */
const DAY_START_HOUR = 6;
const NIGHT_START_HOUR = 18;
/** How long a place-name lookup may hold up an Open-Meteo refresh. */
const GEOCODE_TIMEOUT_MS = 10_000;
const HOUR_MS = 60 * 60 * 1000;
const DAY_MS = 24 * HOUR_MS;
const WEEKDAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
/** Refresh period while the Weather app is open. */
const APP_REFRESH_MS = 30 * 60 * 1000;
/** Refresh period for the status-bar and system-card indicators. */
@@ -127,6 +184,7 @@ const CACHE_VERSION = 1;
const DEFAULT_STATE: WeatherState = {
phase: "permission-required",
status: "Location permission is required for local weather.",
source: null,
locationName: "",
current: null,
forecast: [],
@@ -135,6 +193,8 @@ const DEFAULT_STATE: WeatherState = {
type WeatherCache = {
version: number;
/** Absent from caches written before Open-Meteo, which all came from NWS. */
source?: WeatherSource;
locationName: string;
current: CurrentWeather | null;
forecast: ForecastPeriod[];
@@ -262,12 +322,13 @@ export class WeatherBridge {
this.emit();
const location = await getCurrentLocation();
this.state = { ...this.state, phase: "loading", status: "Loading National Weather Service data..." };
this.state = { ...this.state, phase: "loading", status: "Loading weather..." };
this.emit();
const weather = await loadNwsWeather(location);
const weather = await loadWeather(location);
this.state = {
phase: "ready",
status: "Weather updated.",
source: weather.source,
locationName: weather.locationName,
current: weather.current,
forecast: weather.forecast,
@@ -311,6 +372,7 @@ function loadCachedState(): WeatherState {
return cloneState({
phase: "ready",
status: "Weather updated.",
source: cache.source === "open-meteo" ? "open-meteo" : "nws",
locationName: typeof cache.locationName === "string" ? cache.locationName : "",
current: cache.current && typeof cache.current === "object" ? cache.current : null,
forecast: cache.forecast,
@@ -326,6 +388,7 @@ function saveCachedState(state: WeatherState): void {
if (state.lastUpdatedMs === null) return;
const cache: WeatherCache = {
version: CACHE_VERSION,
source: state.source ?? undefined,
locationName: state.locationName,
current: state.current,
forecast: state.forecast,
@@ -338,15 +401,23 @@ function saveCachedState(state: WeatherState): void {
}
}
async function loadNwsWeather(location: CurrentLocation): Promise<{
locationName: string;
current: CurrentWeather;
forecast: ForecastPeriod[];
}> {
/** NWS where it has coverage (the US and its territories), Open-Meteo everywhere else. */
async function loadWeather(location: CurrentLocation): Promise<LoadedWeather> {
return (await loadNwsWeather(location)) ?? loadOpenMeteoWeather(location);
}
/** NWS weather, or null when the location is outside its coverage. */
async function loadNwsWeather(location: CurrentLocation): Promise<LoadedWeather | null> {
// NWS recommends no more than four decimals for its point lookup.
const latitude = location.latitude.toFixed(4);
const longitude = location.longitude.toFixed(4);
const point = await fetchNwsJson<NwsPointResponse>(`${NWS_API_ROOT}/points/${latitude},${longitude}`);
let point: NwsPointResponse;
try {
point = await fetchNwsJson<NwsPointResponse>(`${NWS_API_ROOT}/points/${latitude},${longitude}`);
} catch (error) {
if (error instanceof WeatherHttpError && error.status === 404) return null;
throw error;
}
const properties = point.properties;
const forecastUrl = stringValue(properties?.forecast);
const hourlyUrl = stringValue(properties?.forecastHourly);
@@ -369,6 +440,7 @@ async function loadNwsWeather(location: CurrentLocation): Promise<{
.find((period): period is ForecastPeriod => period !== null);
return {
source: "nws",
locationName: pointLocationName(point, latitude, longitude),
current: observation ? normalizeObservation(observation, hourly) : currentFromHourly(hourly, forecast[0]!),
forecast,
@@ -382,20 +454,26 @@ async function loadLatestObservation(stationsUrl: string): Promise<NwsObservatio
return fetchNwsJson<NwsObservationResponse>(`${stationUrl}/observations/latest`);
}
async function fetchNwsJson<T>(url: string): Promise<T> {
const request = fetchWithUserAgent(url, { headers: NWS_HEADERS });
function fetchNwsJson<T>(url: string): Promise<T> {
return fetchWeatherJson<T>(url, "National Weather Service", NWS_HEADERS);
}
/** A weather request the server answered with an error status. */
class WeatherHttpError extends Error {
constructor(sourceName: string, readonly status: number) {
super(`${sourceName} request failed (HTTP ${status}).`);
}
}
async function fetchWeatherJson<T>(url: string, sourceName: string, headers?: Record<string, string>): Promise<T> {
const request = fetchWithUserAgent(url, headers ? { headers } : undefined);
let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
const timeout = new Promise<never>((_resolve, reject) => {
timeoutHandle = setTimeout(() => reject(new Error("Weather request timed out.")), FETCH_TIMEOUT_MS);
});
try {
const response = await Promise.race([request, timeout]);
if (!response.ok) {
if (response.status === 404 && url.includes("/points/")) {
throw new Error("This location is outside National Weather Service coverage.");
}
throw new Error(`National Weather Service request failed (HTTP ${response.status}).`);
}
if (!response.ok) throw new WeatherHttpError(sourceName, response.status);
return (await response.json()) as T;
} finally {
if (timeoutHandle) clearTimeout(timeoutHandle);
@@ -526,6 +604,193 @@ function conditionFromText(text: string): WeatherCondition | null {
return null;
}
async function loadOpenMeteoWeather(location: CurrentLocation): Promise<LoadedWeather> {
const latitude = location.latitude.toFixed(4);
const longitude = location.longitude.toFixed(4);
const query = [
`latitude=${latitude}`,
`longitude=${longitude}`,
"current=temperature_2m,relative_humidity_2m,is_day,weather_code,wind_speed_10m,wind_direction_10m",
"hourly=temperature_2m,precipitation_probability,weather_code,wind_speed_10m,wind_direction_10m",
`forecast_days=${OPEN_METEO_FORECAST_DAYS}`,
// Local-time periods need the location's UTC offset, which timezone=auto adds.
"timezone=auto",
"timeformat=unixtime",
"temperature_unit=fahrenheit",
"wind_speed_unit=mph",
].join("&");
const [response, placeName] = await Promise.all([
fetchWeatherJson<OpenMeteoResponse>(`${OPEN_METEO_FORECAST_URL}?${query}`, "Open-Meteo"),
lookUpPlaceName(location),
]);
const forecast = openMeteoForecastPeriods(response, Date.now()).slice(0, MAX_FORECAST_PERIODS);
if (!forecast.length) throw new Error("Open-Meteo returned an empty forecast.");
return {
source: "open-meteo",
locationName: placeName || `${latitude}, ${longitude}`,
current: normalizeOpenMeteoCurrent(response) ?? currentFromHourly(undefined, forecast[0]!),
forecast,
};
}
/** Open-Meteo's current conditions, which come from its forecast models rather than a station. */
function normalizeOpenMeteoCurrent(response: OpenMeteoResponse): CurrentWeather | null {
const current = response.current;
if (!current || typeof current !== "object") return null;
const wmo = wmoCode(finiteNumber(current.weather_code));
const windDegrees = finiteNumber(current.wind_direction_10m);
const time = finiteNumber(current.time);
const isDay = finiteNumber(current.is_day);
return {
temperatureF: finiteNumber(current.temperature_2m),
description: wmo?.description ?? "Current conditions",
humidityPercent: finiteNumber(current.relative_humidity_2m),
windSpeedMph: finiteNumber(current.wind_speed_10m),
windDirection: windDegrees === null ? "" : degreesToCompass(windDegrees),
timestampMs: time === null ? null : time * 1000,
observed: false,
condition: wmo?.condition ?? null,
isDaytime: isDay === null ? isLocalDaytime() : isDay === 1,
};
}
/**
* NWS-style day (6am-6pm) and night periods from Open-Meteo's hourly
* forecast, in the location's local time: each has its warmest daytime or
* coolest nighttime temperature, its highest chance of rain, and the weather
* from periodWeatherCode. The current period counts only the hours left in
* it. Local time uses today's UTC offset, so periods after a DST change are
* an hour off.
*/
function openMeteoForecastPeriods(response: OpenMeteoResponse, nowMs: number): ForecastPeriod[] {
const hourly = response.hourly;
const times = numberArray(hourly?.time);
const temperatures = numberArray(hourly?.temperature_2m);
const precipitation = numberArray(hourly?.precipitation_probability);
const codes = numberArray(hourly?.weather_code);
const windSpeeds = numberArray(hourly?.wind_speed_10m);
const windDirections = numberArray(hourly?.wind_direction_10m);
const offsetMs = (finiteNumber(response.utc_offset_seconds) ?? 0) * 1000;
const todayIndex = Math.floor((nowMs + offsetMs) / DAY_MS);
// The hourly indexes of each period still to come, in order.
const groups: Array<{ dayIndex: number; isDaytime: boolean; hours: number[] }> = [];
times.forEach((time, index) => {
if (time === null || time * 1000 + HOUR_MS <= nowMs) return;
const localMs = time * 1000 + offsetMs;
const hour = new Date(localMs).getUTCHours();
const isDaytime = hour >= DAY_START_HOUR && hour < NIGHT_START_HOUR;
// A night belongs to the date it starts on.
const dayIndex = Math.floor(localMs / DAY_MS) - (hour < DAY_START_HOUR ? 1 : 0);
const last = groups[groups.length - 1];
if (last && last.dayIndex === dayIndex && last.isDaytime === isDaytime) last.hours.push(index);
else groups.push({ dayIndex, isDaytime, hours: [index] });
});
return groups.map(({ dayIndex, isDaytime, hours }) => {
const during = (values: Array<number | null>) => hours.map((index) => values[index] ?? null);
const wmo = wmoCode(periodWeatherCode(during(codes)));
const speeds = during(windSpeeds);
const windiest = maxValue(speeds);
const windDegrees = windiest === null ? null : windDirections[hours[speeds.indexOf(windiest)]!] ?? null;
return {
name: openMeteoPeriodName(dayIndex, isDaytime, todayIndex),
startTimeMs: times[hours[0]!]! * 1000,
temperatureF: isDaytime ? maxValue(during(temperatures)) : minValue(during(temperatures)),
shortForecast: wmo?.description ?? "",
detailedForecast: "",
precipitationPercent: maxValue(during(precipitation)),
windSpeed: windiest === null ? "" : `${Math.round(windiest)} mph`,
windDirection: windDegrees === null ? "" : degreesToCompass(windDegrees),
isDaytime,
condition: wmo?.condition ?? null,
};
});
}
/**
* The most severe weather code that lasts at least two hours of a period (or
* its only hour), so one stray hour of drizzle doesn't name it. WMO codes
* rise with severity, as Open-Meteo's own daily summaries assume.
*/
function periodWeatherCode(codes: Array<number | null>): number | null {
const known = codes.filter((code): code is number => code !== null).sort((a, b) => b - a);
return known.length ? known[Math.min(1, known.length - 1)]! : null;
}
/** "Today", "Tonight", "Friday", "Friday Night", or "Overnight" for last night's remainder. */
function openMeteoPeriodName(dayIndex: number, isDaytime: boolean, todayIndex: number): string {
if (dayIndex < todayIndex) return "Overnight";
if (dayIndex === todayIndex) return isDaytime ? "Today" : "Tonight";
const weekday = WEEKDAYS[new Date(dayIndex * DAY_MS).getUTCDay()]!;
return isDaytime ? weekday : `${weekday} Night`;
}
// WMO weather interpretation codes, as Open-Meteo reports them
// (https://open-meteo.com/en/docs#weather_variable_documentation).
const WMO_CODES: Record<number, { description: string; condition: WeatherCondition }> = {
0: { description: "Clear", condition: "clear" },
1: { description: "Mostly Clear", condition: "partly-cloudy" },
2: { description: "Partly Cloudy", condition: "partly-cloudy" },
3: { description: "Overcast", condition: "cloudy" },
45: { description: "Fog", condition: "fog" },
48: { description: "Freezing Fog", condition: "fog" },
51: { description: "Light Drizzle", condition: "drizzle" },
53: { description: "Drizzle", condition: "drizzle" },
55: { description: "Heavy Drizzle", condition: "drizzle" },
56: { description: "Light Freezing Drizzle", condition: "sleet" },
57: { description: "Freezing Drizzle", condition: "sleet" },
61: { description: "Light Rain", condition: "rain" },
63: { description: "Rain", condition: "rain" },
65: { description: "Heavy Rain", condition: "rain" },
66: { description: "Light Freezing Rain", condition: "sleet" },
67: { description: "Freezing Rain", condition: "sleet" },
71: { description: "Light Snow", condition: "snow" },
73: { description: "Snow", condition: "snow" },
75: { description: "Heavy Snow", condition: "snow" },
77: { description: "Snow Grains", condition: "snow" },
80: { description: "Light Rain Showers", condition: "showers" },
81: { description: "Rain Showers", condition: "showers" },
82: { description: "Heavy Rain Showers", condition: "rain" },
85: { description: "Light Snow Showers", condition: "snow" },
86: { description: "Heavy Snow Showers", condition: "snow" },
95: { description: "Thunderstorms", condition: "thunderstorm" },
96: { description: "Thunderstorms and Hail", condition: "thunderstorm" },
99: { description: "Thunderstorms and Heavy Hail", condition: "thunderstorm" },
};
function wmoCode(code: number | null): { description: string; condition: WeatherCondition } | undefined {
return code === null ? undefined : WMO_CODES[code];
}
/** "Sydney, Australia" from the phone's geocoder, or "" when it can't name the place in time. */
async function lookUpPlaceName(location: CurrentLocation): Promise<string> {
let timeoutHandle: ReturnType<typeof setTimeout> | null = null;
const timeout = new Promise<null>((resolve) => {
timeoutHandle = setTimeout(() => resolve(null), GEOCODE_TIMEOUT_MS);
});
try {
const place = await Promise.race([
reverseGeocode(location.latitude, location.longitude).catch((error) => {
console.warn(`weather place lookup failed: ${error}`);
return null;
}),
timeout,
]);
return placeLabel(place);
} finally {
if (timeoutHandle) clearTimeout(timeoutHandle);
}
}
/** The city (or failing that, the region) and country. */
function placeLabel(place: Place | null): string {
if (!place) return "";
const local = place.locality || place.region;
if (!local || local === place.country) return place.country;
return place.country ? `${local}, ${place.country}` : local;
}
/** Rough day/night by the phone's clock, for when NWS gives no hint. */
function isLocalDaytime(): boolean {
const hour = new Date().getHours();
@@ -573,6 +838,20 @@ function finiteNumber(value: unknown): number | null {
return Number.isFinite(number) ? number : null;
}
function numberArray(value: unknown): Array<number | null> {
return Array.isArray(value) ? value.map((item) => finiteNumber(item)) : [];
}
function maxValue(values: Array<number | null>): number | null {
const numbers = values.filter((value): value is number => value !== null);
return numbers.length ? Math.max(...numbers) : null;
}
function minValue(values: Array<number | null>): number | null {
const numbers = values.filter((value): value is number => value !== null);
return numbers.length ? Math.min(...numbers) : null;
}
function timestampValue(value: unknown): number | null {
if (typeof value !== "string") return null;
const timestamp = Date.parse(value);
@@ -586,7 +865,7 @@ function stringValue(value: unknown): string {
function friendlyWeatherError(error: unknown): string {
const message = (error as Error)?.message || String(error);
if (/network request failed|failed to fetch|unable to resolve host/i.test(message)) {
return "Couldn't reach the National Weather Service. Check the phone's connection and retry.";
return "Couldn't reach the weather service. Check the phone's connection and retry.";
}
return message;
}
+456
View File
@@ -0,0 +1,456 @@
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
/**
* Speech-to-text against a self-hosted OpenAI-compatible Whisper server (vLLM,
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
* is one request, finalized in audio order; an open segment is re-sent every
* INTERIM_STEP_BYTES for provisional text. A segment that runs SEGMENT_MAX_BYTES
* without a pause is cut into requests that still leave as one final. start()
* lists {baseUrl}/models to check reachability and pick a model when none is
* configured.
*/
export type WhisperServerConfig = {
/** OpenAI-style API root, e.g. http://host:8004/v1. */
baseUrl: string;
/** Model id to request; empty uses the first one the server lists. */
model: string;
/** ISO-639-1 language code; empty lets Whisper detect it. */
language: string;
};
export type WhisperHttpRequest = {
url: string;
method: "GET" | "POST";
headers: Record<string, string>;
body?: ArrayBuffer;
timeoutMs: number;
};
export type WhisperHttpResponse = { status: number; text: string };
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
// Provisional transcripts: first after one second of audio, then every 1.5s.
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
// Steady speech or background noise can keep the pause detector from ever
// ending a span. Such a segment is cut at this length, inside Whisper's 30s
// window and well within FINAL_TIMEOUT_MS, after its quietest chunk among the
// last SPLIT_SEARCH_BYTES so a word is less likely to straddle the cut.
const SEGMENT_MAX_BYTES = BYTES_PER_SECOND * 25;
const SPLIT_SEARCH_BYTES = BYTES_PER_SECOND * 5;
const MODELS_TIMEOUT_MS = 5000;
const INTERIM_TIMEOUT_MS = 10000;
const FINAL_TIMEOUT_MS = 12000;
const FINAL_RETRY_DELAY_MS = 750;
// Whisper invents text ("Thank you.") for silence, so a segment whose loudest
// 50ms frame stays below this RMS is not sent at all. Normal speech on the G2
// mic measures in the thousands; the pause detector's noise floor starts at 220.
const SILENCE_RMS = 300;
/** Time the voice bridges should allow a final transcript before giving up. */
export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETRY_DELAY_MS + 2000;
type Segment = {
chunks: Uint8Array[];
/** RMS of each chunk. */
levels: number[];
bytes: number;
peakRms: number;
text: string;
final: boolean;
closed: boolean;
paragraphBreakAfter: boolean;
/** Emit a final even when empty: the end of push-to-talk needs one. */
required: boolean;
/** Cut for length: its final leaves joined with the next segment's. */
continues: boolean;
interimInFlight: boolean;
interimBytes: number;
};
/**
* The API root from what a user typed: tolerates a trailing slash or the full
* transcription endpoint, and adds /v1 to a bare host. Empty when unusable.
*/
export function normalizeWhisperServerUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
if (!/^https?:\/\/[^/\s]+/i.test(url)) return "";
url = url.replace(/\/audio\/transcriptions$/i, "");
if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1";
return url;
}
export class WhisperServerSttClient implements CloudSttClient {
private readonly baseUrl: string;
private model: string;
private ready = false;
private closed = false;
private readonly segments: Segment[] = [];
constructor(
private readonly options: CloudSttOptions,
private readonly config: WhisperServerConfig,
private readonly http: WhisperHttp = nativeHttp,
) {
this.baseUrl = normalizeWhisperServerUrl(config.baseUrl);
this.model = config.model.trim();
}
start(): void {
if (this.closed || this.ready) return;
if (!this.baseUrl) {
this.options.onError("Whisper server URL is not a valid http(s) address.");
return;
}
this.options.onStatus("Connecting to Whisper server...");
this.http({ url: `${this.baseUrl}/models`, method: "GET", headers: this.authHeaders(), timeoutMs: MODELS_TIMEOUT_MS })
.then((response) => {
if (this.closed) return;
if (response.status < 200 || response.status >= 300) {
this.options.onDisconnected?.(`Whisper server returned ${response.status} for /models.`);
return;
}
if (!this.model) this.model = firstModelId(response.text);
if (!this.model) {
this.options.onError("Whisper server lists no models; set the model in Settings > Voice.");
return;
}
this.ready = true;
this.options.onStatus("Listening (Whisper server)...");
this.options.onReady?.();
})
.catch((error) => {
if (!this.closed) this.options.onDisconnected?.(`Whisper server unreachable: ${describeError(error)}`);
});
}
/** PCM16LE at 16 kHz, appended to the open segment. */
acceptPcm(pcm: Uint8Array): void {
if (this.closed || !pcm.length) return;
const segment = this.openSegment();
const level = rms(pcm);
segment.chunks.push(pcm.slice());
segment.levels.push(level);
segment.bytes += pcm.length;
segment.peakRms = Math.max(segment.peakRms, level);
if (segment.bytes >= SEGMENT_MAX_BYTES) this.splitSegment(segment);
this.maybeRequestInterim(this.segments[this.segments.length - 1]!);
}
/** A pause in continuous capture: transcribe what came before it. */
commitSegment(): void {
this.closeSegment(true, false);
}
/** End of the utterance: transcribe the rest and deliver a final. */
finish(): void {
this.closeSegment(false, true);
}
stop(): void {
this.closed = true;
this.segments.length = 0;
}
private openSegment(): Segment {
const last = this.segments[this.segments.length - 1];
if (last && !last.closed) return last;
const segment: Segment = {
chunks: [], levels: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
paragraphBreakAfter: false, required: false, continues: false, interimInFlight: false, interimBytes: 0,
};
this.segments.push(segment);
return segment;
}
/**
* Close the open segment at SEGMENT_MAX_BYTES and carry the audio after its
* cut into a new one. The cut is after the quietest chunk among the last
* SPLIT_SEARCH_BYTES; the first chunk always stays.
*/
private splitSegment(segment: Segment): void {
let cut = segment.chunks.length - 1;
let quietest = Infinity;
let carried = 0;
for (let i = segment.chunks.length - 1; i > 0 && carried <= SPLIT_SEARCH_BYTES; i--) {
if (segment.levels[i]! < quietest) {
quietest = segment.levels[i]!;
cut = i;
}
carried += segment.chunks[i]!.length;
}
const chunks = segment.chunks.splice(cut + 1);
const levels = segment.levels.splice(cut + 1);
for (const chunk of chunks) segment.bytes -= chunk.length;
segment.peakRms = peak(segment.levels);
segment.continues = true;
this.closeSegment(false, false);
if (!chunks.length) return;
const next = this.openSegment();
next.chunks = chunks;
next.levels = levels;
for (const chunk of chunks) next.bytes += chunk.length;
next.peakRms = peak(levels);
}
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
if (this.closed) return;
const last = this.segments[this.segments.length - 1];
if (last?.closed && last.continues) {
// Cut for length with nothing after it yet: this boundary ends its run.
last.continues = false;
last.paragraphBreakAfter = paragraphBreakAfter;
last.required = required;
if (last.final) this.emit();
return;
}
if (!last || last.closed) {
// Nothing new since the last boundary. Push-to-talk still needs a final.
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
return;
}
if (last.peakRms < SILENCE_RMS) {
this.finalize(last, "", paragraphBreakAfter, required);
return;
}
last.closed = true;
last.paragraphBreakAfter = paragraphBreakAfter;
last.required = required;
void this.requestFinal(last, 0);
}
private async requestFinal(segment: Segment, attempt: number): Promise<void> {
try {
const text = await this.transcribe(segment, segment.bytes, FINAL_TIMEOUT_MS);
if (!this.closed) this.finalize(segment, text, segment.paragraphBreakAfter, segment.required);
} catch (error) {
if (this.closed) return;
if (error instanceof WhisperAuthError) {
this.options.onError(error.message);
return;
}
if (attempt === 0) {
setTimeout(() => { void this.requestFinal(segment, 1); }, FINAL_RETRY_DELAY_MS);
return;
}
// Keep the capture alive and keep whatever the interim pass heard.
this.options.onStatus(`Whisper server request failed: ${describeError(error)}`);
this.finalize(segment, segment.text, segment.paragraphBreakAfter, segment.required);
}
}
private maybeRequestInterim(segment: Segment): void {
if (!this.ready || segment.interimInFlight || segment.closed) return;
if (segment.bytes < INTERIM_MIN_BYTES) return;
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
if (segment.peakRms < SILENCE_RMS) return;
segment.interimInFlight = true;
segment.interimBytes = segment.bytes;
this.transcribe(segment, segment.bytes, INTERIM_TIMEOUT_MS)
.then((text) => {
if (this.closed || segment.final) return;
segment.text = text;
this.emit();
})
.catch(() => {
// Provisional only; the final request reports real failures.
})
.finally(() => {
segment.interimInFlight = false;
});
}
private finalize(segment: Segment, text: string, paragraphBreakAfter: boolean, required: boolean): void {
segment.closed = segment.final = true;
segment.text = text;
segment.paragraphBreakAfter = paragraphBreakAfter;
segment.required = required;
segment.chunks = [];
this.emit();
}
/**
* Finals leave in audio order, even when a later request answers first, and
* segments cut for length leave as one final with the segment that ends
* their run; whatever is not final yet is shown as one provisional transcript.
*/
private emit(): void {
for (;;) {
let end = 0;
while (this.segments[end]?.final && this.segments[end]!.continues) end++;
if (!this.segments[end]?.final) break;
const run = this.segments.splice(0, end + 1);
const last = run[run.length - 1]!;
const text = run.map((segment) => segment.text.trim()).filter(Boolean).join(" ");
if (!text && !last.required) continue;
this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: last.paragraphBreakAfter });
}
const pending = this.segments.filter((segment) => segment.text.trim());
if (!pending.length) return;
const text = pending.map((segment) => segment.text.trim()).join(" ");
const transcribeText = pending.map((segment, index) =>
(index && pending[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + segment.text.trim(),
).join("");
this.options.onTranscript({ text, transcribeText, isFinal: false });
}
private async transcribe(segment: Segment, bytes: number, timeoutMs: number): Promise<string> {
const fields: Record<string, string> = { model: this.model, response_format: "json", temperature: "0" };
const language = this.config.language.trim().toLowerCase();
if (language) fields.language = language;
const boundary = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
const body = multipartBody(boundary, fields, wavFile(segment.chunks, bytes));
const response = await this.http({
url: `${this.baseUrl}/audio/transcriptions`,
method: "POST",
headers: { ...this.authHeaders(), "Content-Type": `multipart/form-data; boundary=${boundary}` },
body,
timeoutMs,
});
if (response.status === 401 || response.status === 403) {
throw new WhisperAuthError(`Whisper server rejected the key (${response.status}).`);
}
if (response.status < 200 || response.status >= 300) {
throw new Error(`HTTP ${response.status}: ${response.text.slice(0, 200)}`);
}
return parseTranscript(response.text);
}
private authHeaders(): Record<string, string> {
const key = this.options.apiKey.trim();
return key ? { Authorization: `Bearer ${key}` } : {};
}
}
class WhisperAuthError extends Error {}
/** The transcript from a JSON response; tolerates servers that send plain text. */
export function parseTranscript(body: string): string {
const raw = body.trim();
try {
const parsed = JSON.parse(raw);
if (parsed && typeof parsed.text === "string") return parsed.text.trim();
} catch {
// Not JSON: a server that answered response_format=text anyway.
}
return raw;
}
function firstModelId(body: string): string {
try {
const parsed = JSON.parse(body);
const id = parsed?.data?.[0]?.id;
return typeof id === "string" ? id : "";
} catch {
return "";
}
}
function peak(levels: number[]): number {
let max = 0;
for (const level of levels) max = Math.max(max, level);
return max;
}
/** RMS of a PCM16LE chunk. */
function rms(pcm: Uint8Array): number {
const count = pcm.length >> 1;
if (!count) return 0;
let squares = 0;
for (let i = 0; i < count; i++) {
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
squares += sample * sample;
}
return Math.sqrt(squares / count);
}
/** The first `bytes` of the chunks as a 16 kHz mono 16-bit WAV file. */
export function wavFile(chunks: Uint8Array[], bytes: number): Uint8Array {
const length = bytes & ~1;
const out = new Uint8Array(44 + length);
const view = new DataView(out.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) out[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + length, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, CLOUD_STT_SAMPLE_RATE, true);
view.setUint32(28, BYTES_PER_SECOND, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
ascii(36, "data");
view.setUint32(40, length, true);
let offset = 44;
for (const chunk of chunks) {
const take = Math.min(chunk.length, out.length - offset);
if (take <= 0) break;
out.set(chunk.subarray(0, take), offset);
offset += take;
}
return out;
}
/** multipart/form-data with text fields and one WAV part named "file". */
export function multipartBody(boundary: string, fields: Record<string, string>, wav: Uint8Array): ArrayBuffer {
let head = "";
for (const [name, value] of Object.entries(fields)) {
head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`;
}
head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`;
const headBytes = utf8(head);
const tailBytes = utf8(`\r\n--${boundary}--\r\n`);
const out = new Uint8Array(headBytes.length + wav.length + tailBytes.length);
out.set(headBytes, 0);
out.set(wav, headBytes.length);
out.set(tailBytes, headBytes.length + wav.length);
return out.buffer;
}
function utf8(text: string): Uint8Array {
const out: number[] = [];
for (const char of text) {
const code = char.codePointAt(0)!;
if (code < 0x80) out.push(code);
else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 63));
else if (code < 0x10000) out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
else out.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
}
return Uint8Array.from(out);
}
function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/**
* Uploads go through postBinary (on Android NativeScript's HTTP module would
* copy each body into Java a byte at a time); the model list through
* NativeScript's HTTP module.
*/
const nativeHttp: WhisperHttp = async (request) => {
// Required lazily so Node tests can load this module without NativeScript.
if (request.body) {
const { postBinary } = require("./binary-http") as typeof import("./binary-http");
return postBinary(request.url, request.body, request.headers, request.timeoutMs);
}
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
const response = await send({
url: request.url,
method: request.method,
headers: withUserAgent(request.headers),
timeout: request.timeoutMs,
});
const content = response.content as any;
let text = "";
if (content) text = global.isAndroid ? String(content.raw.toString("UTF-8")) : String(content.toString());
return { status: response.statusCode, text };
};