forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes
This commit is contained in:
commit
fcea0b369a
218 files changed
+14438
-2259
No files matched your search
+54
@@ -154,6 +154,60 @@ Button.-primary {
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
/* Keyboard-input panel: tighter than the setting editor, with the text
|
||||
* box's edges in line with the title and the one-strip buttons (the theme's
|
||||
* 16dp TextView side margins doubled the inset). */
|
||||
.system-name-modal.keyboard-input-panel {
|
||||
margin: 8;
|
||||
padding: 8 16 12 16;
|
||||
}
|
||||
|
||||
.keyboard-input-panel TextView {
|
||||
margin: 4 0 0 0;
|
||||
padding: 6;
|
||||
}
|
||||
|
||||
/* Glasses-only: the field stays focused for the IME but is an invisible
|
||||
* sliver; the hint line takes its place. */
|
||||
.keyboard-input-panel TextView.keyboard-input-field-hidden {
|
||||
opacity: 0;
|
||||
height: 1;
|
||||
min-height: 0;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.system-name-modal Label.keyboard-input-hidden-hint,
|
||||
.ns-dark .system-name-modal Label.keyboard-input-hidden-hint {
|
||||
color: #666666;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.keyboard-input-buttons {
|
||||
margin-top: 8;
|
||||
}
|
||||
|
||||
.keyboard-input-panel Button,
|
||||
.keyboard-input-panel Button.-primary {
|
||||
margin: 0;
|
||||
font-size: 16;
|
||||
}
|
||||
|
||||
.keyboard-input-panel Button.keyboard-input-gap {
|
||||
margin-left: 8;
|
||||
}
|
||||
|
||||
/* Flat icon buttons in the panel's title row (the modal stays light in dark
|
||||
* mode, so one tint). */
|
||||
.panel-icon-button {
|
||||
padding: 6 8;
|
||||
}
|
||||
|
||||
.panel-icon {
|
||||
height: 20;
|
||||
tint-color: #333333;
|
||||
}
|
||||
|
||||
/* Modal behind the action bar's warning triangle: the problem-status warnings
|
||||
* that used to sit inline on the main page, each with its fix-it button.
|
||||
* The width is set from the view model (near-full-width on phones, capped on
|
||||
|
||||
@@ -56,6 +56,9 @@ Page.main-page {
|
||||
color: #222222;
|
||||
padding: 16;
|
||||
}
|
||||
/* The modal stays light in dark mode; outgun the theme's `.ns-dark .h2`
|
||||
* (white) so the title doesn't go white-on-light-gray. */
|
||||
.main-page .system-name-modal .h2 { color: #222222; }
|
||||
.main-page .system-name-modal TextView {
|
||||
margin: 0;
|
||||
padding: 4;
|
||||
@@ -67,3 +70,15 @@ Page.main-page {
|
||||
.main-landscape .system-name-modal { padding: 6 12; }
|
||||
.main-landscape .system-name-modal .h2 { visibility: collapse; }
|
||||
.main-landscape .system-name-modal Button { padding: 6 8; margin-top: 0; margin-bottom: 0; }
|
||||
|
||||
/* Keyboard-input panel: the one-strip buttons and title-row icons (see app.css). */
|
||||
.main-page .system-name-modal.keyboard-input-panel { padding: 8 16 12 16; }
|
||||
.main-landscape .system-name-modal.keyboard-input-panel { padding: 6 12; }
|
||||
.main-page .keyboard-input-panel TextView { margin: 4 0 0 0; padding: 6; }
|
||||
.main-page .keyboard-input-panel TextView.keyboard-input-field-hidden { opacity: 0; height: 1; min-height: 0; margin: 0; padding: 0; }
|
||||
.main-page .keyboard-input-hidden-hint { color: #666666; padding: 8 0; }
|
||||
.main-page .keyboard-input-buttons { margin-top: 8; }
|
||||
.main-page .keyboard-input-panel Button { margin: 0; }
|
||||
.main-page .keyboard-input-panel Button.keyboard-input-gap { margin-left: 8; }
|
||||
.main-page .panel-icon-button { padding: 6 8; }
|
||||
.main-page .panel-icon { height: 20; tint-color: #333333; }
|
||||
@@ -4,7 +4,7 @@ import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { ASSISTANT_MODEL_CHOICES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, assistantOpenAiCompatibleKeySetting, assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext } from "../../ui/layers";
|
||||
import type { MenuItem } from "../../ui/menu";
|
||||
@@ -102,7 +102,7 @@ class AiChatLayer implements Layer {
|
||||
}
|
||||
case "long-press":
|
||||
if (!this.conversations.current().session?.isTurnActive()) {
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings."); return; }
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(shell.assistantSetupHint()); return; }
|
||||
void this.draft.start();
|
||||
}
|
||||
return;
|
||||
@@ -133,7 +133,12 @@ class AiChatLayer implements Layer {
|
||||
onSelect: (pickCtx) => { if (this.conversations.configure(model, record.reasoning)) pickCtx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||
const llm = resolveAssistantModel(record.model, {
|
||||
anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
return external || busy() || !llm?.effort;
|
||||
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||
|
||||
@@ -7,13 +7,17 @@
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantModelSetting,
|
||||
assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting,
|
||||
assistantOpenAiCompatibleUrlSetting,
|
||||
openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
|
||||
@@ -22,6 +26,9 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
if (!resolved) return Promise.resolve(null);
|
||||
|
||||
@@ -40,8 +47,11 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
case "openai":
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "openai-compatible":
|
||||
streamOpenAiCompatibleChat({ ...options, baseUrl: resolved.baseUrl ?? "" });
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
streamLocalModel(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
|
||||
@@ -14,6 +14,9 @@ import {
|
||||
const MESSAGE_SIZES = [250, 500, 1000, 2000, 3800];
|
||||
const WINDOW_SIZES = [1, 2, 3, 4, 6];
|
||||
const LINK_MODES = ["Current", "Request HIGH", "Request 2M", "HIGH + 2M"];
|
||||
// CoreBluetooth has no connection-priority or PHY request API (iOS's
|
||||
// prepareBenchmarkLink is a no-op), so iOS always runs on the current link.
|
||||
const LINK_REQUESTS_SUPPORTED = !global.isIOS;
|
||||
const DURATION_MS = 15_000;
|
||||
// After painting the "running" screen, wait for that frame to reach the
|
||||
// glasses before the benchmark blocks normal image sends — otherwise the
|
||||
@@ -40,7 +43,7 @@ type ConfigRow = "size" | "window" | "link" | "start";
|
||||
export class BandwidthBenchmarkLayer implements Layer {
|
||||
private phase: Phase = "config";
|
||||
private readonly menu = new Menu<ConfigRow>({
|
||||
items: ["size", "window", "link", "start"],
|
||||
items: LINK_REQUESTS_SUPPORTED ? ["size", "window", "link", "start"] : ["size", "window", "start"],
|
||||
wrap: false,
|
||||
rowGap: 1,
|
||||
highlight: { radius: 4 },
|
||||
@@ -134,7 +137,8 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
font,
|
||||
LIST_X,
|
||||
HEADER_HEIGHT + 30,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}, ${LINK_MODES[this.linkModeIndex]}`,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}`
|
||||
+ (LINK_REQUESTS_SUPPORTED ? `, ${LINK_MODES[this.linkModeIndex]}` : ""),
|
||||
150,
|
||||
);
|
||||
image.drawText(font, LIST_X, HEADER_HEIGHT + 52, "Display updates pause until the run finishes.", 110);
|
||||
@@ -161,8 +165,11 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
`Throughput: ${(payloadRate / 1024).toFixed(2)} KB/s payload (${(wireRate / 1024).toFixed(2)} KB/s wire)`,
|
||||
`Messages: ${r.messagesAcked}/${r.messagesSent} acked, ${r.timeouts} timeouts`,
|
||||
`Elapsed: ${seconds.toFixed(1)} s` + (r.aborted ? " (aborted early)" : ""),
|
||||
`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}`,
|
||||
];
|
||||
if (LINK_REQUESTS_SUPPORTED) {
|
||||
const phy = r.leftPhy || r.rightPhy ? `; PHY L ${r.leftPhy ?? "?"}, R ${r.rightPhy ?? "?"}` : "";
|
||||
lines.push(`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}${phy}`);
|
||||
}
|
||||
for (const line of lines) {
|
||||
image.drawText(font, LIST_X, y, line, 190);
|
||||
y += 20;
|
||||
|
||||
@@ -51,24 +51,18 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Buzzer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Glasses sound playback is not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new BuzzerDemoLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Accelerometer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new AccelerometerDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Light sensor",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
@@ -83,8 +77,6 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Microphones",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: openMicrophones,
|
||||
},
|
||||
{
|
||||
@@ -131,8 +123,6 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc
|
||||
},
|
||||
{
|
||||
label: "Show resource usage",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new ResourceUsageLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
|
||||
@@ -4,25 +4,23 @@ import { clamp } from "../../util/numeric-util";
|
||||
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
|
||||
declare const com: any;
|
||||
import {
|
||||
MEMORY_SERIES,
|
||||
formatMb,
|
||||
sampleResourceUsage,
|
||||
type ResourceSample,
|
||||
} from "../../native/resource-usage";
|
||||
|
||||
const SAMPLE_INTERVAL_MS = 1_000;
|
||||
const HISTORY_MS = 60_000;
|
||||
const MAX_SAMPLES = Math.ceil(HISTORY_MS / SAMPLE_INTERVAL_MS) + 2;
|
||||
/** Line brightness per memory series, headline first. */
|
||||
const MEMORY_SHADES = [240, 155, 80];
|
||||
|
||||
type ResourceSample = {
|
||||
atMs: number;
|
||||
rssMb: number;
|
||||
nativeMb: number;
|
||||
javaMb: number;
|
||||
javaCommittedMb: number;
|
||||
cpuPercent: number;
|
||||
threads: number;
|
||||
};
|
||||
type ChartSample = ResourceSample & { cpuPercent: number };
|
||||
|
||||
type Series = {
|
||||
value: (sample: ResourceSample) => number;
|
||||
value: (sample: ChartSample) => number;
|
||||
shade: number;
|
||||
};
|
||||
|
||||
@@ -30,9 +28,10 @@ type Series = {
|
||||
* One-minute process monitor. Sampling deliberately continues while another
|
||||
* window is foreground so switching back reveals the resource cost of that
|
||||
* workload. Repaints remain visibility-gated to avoid creating hidden frames.
|
||||
* Which memory figures are charted is up to the platform (MEMORY_SERIES).
|
||||
*/
|
||||
export class ResourceUsageLayer implements Layer {
|
||||
private readonly samples: ResourceSample[] = [];
|
||||
private readonly samples: ChartSample[] = [];
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private removed = false;
|
||||
private previousElapsedMs = 0;
|
||||
@@ -68,34 +67,34 @@ export class ResourceUsageLayer implements Layer {
|
||||
return image;
|
||||
}
|
||||
|
||||
const cpuText = `CPU ${formatPercent(latest.cpuPercent)} ${latest.threads} threads`;
|
||||
image.drawText(font, width - 16 - font.measureText(cpuText), 5, cpuText, 200);
|
||||
image.drawText(font, 16, 21, this.sampleError || latest.status, this.sampleError ? 170 : 140);
|
||||
|
||||
const memorySeries = MEMORY_SERIES.map((label, index) => ({
|
||||
label,
|
||||
shade: MEMORY_SHADES[index] ?? 80,
|
||||
value: (sample: ChartSample) => sample.memoryMb[index] ?? 0,
|
||||
}));
|
||||
const memoryPeak = Math.max(...this.samples.flatMap((sample) => sample.memoryMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
const legendLeft = drawLegend(
|
||||
image,
|
||||
font,
|
||||
width,
|
||||
48,
|
||||
memorySeries.map((series) => [`${series.label} ${formatMb(series.value(latest))}`, series.shade] as const),
|
||||
);
|
||||
const scaleLabel = `Memory 0–${formatMb(memoryScale)}`;
|
||||
const shortScaleLabel = `0–${formatMb(memoryScale)}`;
|
||||
image.drawText(
|
||||
font,
|
||||
16,
|
||||
21,
|
||||
`RSS ${formatMb(latest.rssMb)} native ${formatMb(latest.nativeMb)} Java ${formatMb(latest.javaMb)}/${formatMb(latest.javaCommittedMb)}`,
|
||||
190,
|
||||
48,
|
||||
16 + font.measureText(scaleLabel) + 8 <= legendLeft ? scaleLabel : shortScaleLabel,
|
||||
140,
|
||||
);
|
||||
image.drawText(
|
||||
font,
|
||||
width - 16 - font.measureText(`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`),
|
||||
47 - font.lineHeight,
|
||||
`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`,
|
||||
155,
|
||||
);
|
||||
|
||||
const memoryPeak = Math.max(...this.samples.map((sample) => sample.rssMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
image.drawText(font, 16, 48, `Memory 0–${formatMb(memoryScale)}`, 140);
|
||||
drawLegend(image, font, width, 48, [
|
||||
["RSS", 240],
|
||||
["native", 155],
|
||||
["Java", 80],
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, [
|
||||
{ value: (sample) => sample.rssMb, shade: 240 },
|
||||
{ value: (sample) => sample.nativeMb, shade: 155 },
|
||||
{ value: (sample) => sample.javaMb, shade: 80 },
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, memorySeries);
|
||||
|
||||
const cpuPeak = Math.max(...this.samples.map((sample) => sample.cpuPercent), 100);
|
||||
const cpuScale = roundUp(cpuPeak * 1.08, 25);
|
||||
@@ -119,26 +118,16 @@ export class ResourceUsageLayer implements Layer {
|
||||
private takeSample(): void {
|
||||
if (this.removed) return;
|
||||
try {
|
||||
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
|
||||
const atMs = Number(raw[0]);
|
||||
const cpuMs = Number(raw[1]);
|
||||
const elapsedDelta = atMs - this.previousElapsedMs;
|
||||
const cpuDelta = cpuMs - this.previousCpuMs;
|
||||
const sample = sampleResourceUsage();
|
||||
const elapsedDelta = sample.atMs - this.previousElapsedMs;
|
||||
const cpuDelta = sample.cpuMs - this.previousCpuMs;
|
||||
const cpuPercent = this.previousElapsedMs > 0 && elapsedDelta > 0
|
||||
? Math.max(0, cpuDelta / elapsedDelta * 100)
|
||||
: 0;
|
||||
this.previousElapsedMs = atMs;
|
||||
this.previousCpuMs = cpuMs;
|
||||
this.samples.push({
|
||||
atMs,
|
||||
cpuPercent,
|
||||
rssMb: kbToMb(Number(raw[2])),
|
||||
nativeMb: kbToMb(Number(raw[3])),
|
||||
javaMb: kbToMb(Number(raw[4])),
|
||||
javaCommittedMb: kbToMb(Number(raw[5])),
|
||||
threads: Number(raw[6]),
|
||||
});
|
||||
const cutoff = atMs - HISTORY_MS;
|
||||
this.previousElapsedMs = sample.atMs;
|
||||
this.previousCpuMs = sample.cpuMs;
|
||||
this.samples.push({ ...sample, cpuPercent });
|
||||
const cutoff = sample.atMs - HISTORY_MS;
|
||||
while (this.samples.length > 1 && this.samples[0]!.atMs < cutoff) this.samples.shift();
|
||||
while (this.samples.length > MAX_SAMPLES) this.samples.shift();
|
||||
this.sampleError = "";
|
||||
@@ -153,7 +142,7 @@ export class ResourceUsageLayer implements Layer {
|
||||
|
||||
function drawChart(
|
||||
image: GrayImage,
|
||||
samples: readonly ResourceSample[],
|
||||
samples: readonly ChartSample[],
|
||||
nowMs: number,
|
||||
bounds: { x: number; y: number; width: number; height: number },
|
||||
scale: number,
|
||||
@@ -163,7 +152,8 @@ function drawChart(
|
||||
const midY = bounds.y + Math.round((bounds.height - 1) / 2);
|
||||
image.drawLine(bounds.x + 1, midY, bounds.x + bounds.width - 2, midY, 24);
|
||||
const startMs = nowMs - HISTORY_MS;
|
||||
for (const item of series) {
|
||||
// Back to front, so the headline series is drawn on top where lines cross.
|
||||
for (const item of [...series].reverse()) {
|
||||
let previous: { x: number; y: number } | null = null;
|
||||
for (const sample of samples) {
|
||||
if (sample.atMs < startMs) continue;
|
||||
@@ -177,36 +167,30 @@ function drawChart(
|
||||
}
|
||||
}
|
||||
|
||||
/** Right-aligned legend of line swatches and labels; returns its left edge. */
|
||||
function drawLegend(
|
||||
image: GrayImage,
|
||||
font: ReturnType<typeof getDefaultSmallFont>,
|
||||
width: number,
|
||||
y: number,
|
||||
entries: ReadonlyArray<readonly [string, number]>,
|
||||
): void {
|
||||
): number {
|
||||
const totalWidth = entries.reduce((sum, [label]) => sum + font.measureText(label) + 14, 0);
|
||||
let x = width - 16 - totalWidth;
|
||||
const left = width - 16 - totalWidth;
|
||||
let x = left;
|
||||
for (const [label, shade] of entries) {
|
||||
image.drawLine(x, y + 6, x + 8, y + 6, shade);
|
||||
x += 12;
|
||||
image.drawText(font, x, y, label, shade);
|
||||
x += font.measureText(label) + 2;
|
||||
}
|
||||
}
|
||||
|
||||
function kbToMb(kb: number): number {
|
||||
return Math.max(0, kb) / 1024;
|
||||
return left;
|
||||
}
|
||||
|
||||
function roundUp(value: number, step: number): number {
|
||||
return Math.max(step, Math.ceil(value / step) * step);
|
||||
}
|
||||
|
||||
function formatMb(value: number): string {
|
||||
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
|
||||
return `${Math.round(value)}M`;
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return `${value < 10 ? value.toFixed(1) : Math.round(value)}%`;
|
||||
}
|
||||
@@ -176,6 +176,21 @@ export function readEvenHubPackageManifestBytes(bytes: Uint8Array): EvenHubManif
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an installed EHPK that is all zeros, keeping its registry entry and
|
||||
* icon so the store treats it as missing and offers Reinstall. iOS builds
|
||||
* before the binary-http.ios.ts copy fix could store a large download that
|
||||
* way. Returns whether the package was discarded.
|
||||
*/
|
||||
export function discardZeroedEvenHubPackage(packageId: string): boolean {
|
||||
const path = installedEvenHubPackagePath(packageId);
|
||||
const bytes = readBinaryFile(path);
|
||||
if (!bytes || bytes.length === 0) return false;
|
||||
for (let i = 0; i < bytes.length; i++) if (bytes[i] !== 0) return false;
|
||||
deletePathRecursively(path);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Remove the stored EHPK, unpacked runtime files, and registry entry. */
|
||||
export function uninstallEvenHubPackage(packageId: string): boolean {
|
||||
const apps = getInstalledEvenHubApps();
|
||||
|
||||
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
|
||||
deliverAudioPcm(pcm: Uint8Array): void;
|
||||
};
|
||||
|
||||
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
|
||||
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
* flows (foreground + screen on + no assistant modal).
|
||||
*/
|
||||
private audioControl(data: Record<string, unknown>): boolean {
|
||||
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
|
||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||
return false;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { type GrayImage, type UiFont } from "../../../graphics/image";
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../../native/location-permissions";
|
||||
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
|
||||
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
||||
import { weatherBridge } from "../../../native/weather";
|
||||
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
||||
@@ -37,7 +38,6 @@ import { glanceFont } from "../glance-font";
|
||||
import { type GlanceWidget } from "../widget";
|
||||
|
||||
const PAD = 8;
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const NOTIFICATION_ICON_GAP = 4;
|
||||
const BATTERY_ITEM_GAP = 12;
|
||||
const BATTERY_LABEL_VALUE = 150;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||
|
||||
@@ -245,7 +245,7 @@ export function askConversations(
|
||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
return streamLocalQwen({
|
||||
return streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { useMicControlSetting } from "../../ui/dashboard-settings";
|
||||
import { toJavaBytes } from "../../native/cloud-stt";
|
||||
import { toUint8Array } from "../../util/array-util";
|
||||
import { addMicStatusListener, micControlSession, sendMicControlRecord } from "../../native/mic-control-link";
|
||||
import {
|
||||
MIC_LEASE_MS,
|
||||
decodeMicStatus,
|
||||
@@ -14,8 +13,6 @@ import {
|
||||
type MicStatus,
|
||||
} from "./mic-protocol";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Host-side controller for the CFW mic_control array: sends the same config
|
||||
* to both temples (each temple is its own endpoint and its own 2-mic array),
|
||||
@@ -30,15 +27,6 @@ const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
|
||||
|
||||
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the "Use microphone control" developer setting is on. With the
|
||||
* setting off, Faceclaw behaves exactly as it would on firmware without the
|
||||
@@ -71,9 +59,9 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
// The Java listener proxy must stay referenced or it gets GC'd.
|
||||
private statusProxy: any = null;
|
||||
private proxyCommunicator: any = null;
|
||||
// Holds the native listener proxy, which must stay referenced or it gets GC'd.
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private statusSession: unknown = null;
|
||||
private leaseTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private armed = false;
|
||||
|
||||
@@ -158,39 +146,28 @@ class MicArrayController {
|
||||
rightTemple = true,
|
||||
leftTemple = true,
|
||||
): boolean {
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return false;
|
||||
const session = micControlSession();
|
||||
if (!session) return false;
|
||||
try {
|
||||
this.ensureStatusProxy(communicator);
|
||||
const record = encodeMicControl(op, config);
|
||||
// False when the session wasn't ready and nothing was queued.
|
||||
return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
this.ensureStatusListener(session);
|
||||
return sendMicControlRecord(session, encodeMicControl(op, config), label, rightTemple, leftTemple);
|
||||
} catch (error) {
|
||||
console.warn(`mic control ${label} failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private ensureStatusProxy(communicator: any): void {
|
||||
if (this.statusProxy && this.proxyCommunicator === communicator) return;
|
||||
if (this.statusProxy && this.proxyCommunicator) {
|
||||
try {
|
||||
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
}
|
||||
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
||||
onMicStatus: (body: any, arm: string) => {
|
||||
const status = decodeMicStatus(toUint8Array(body));
|
||||
if (!status) return;
|
||||
const side: MicSide = String(arm) === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
},
|
||||
private ensureStatusListener(session: unknown): void {
|
||||
if (this.unsubscribeStatus && this.statusSession === session) return;
|
||||
this.unsubscribeStatus?.();
|
||||
this.statusSession = session;
|
||||
this.unsubscribeStatus = addMicStatusListener(session, (body, arm) => {
|
||||
const status = decodeMicStatus(body);
|
||||
if (!status) return;
|
||||
const side: MicSide = arm === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
});
|
||||
this.proxyCommunicator = communicator;
|
||||
communicator.addMicStatusListener(this.statusProxy);
|
||||
}
|
||||
|
||||
private syncLeaseTimer(): void {
|
||||
|
||||
@@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment";
|
||||
import { generateSessionInsights } from "./speaker-insights";
|
||||
import { speakerRegistry, type SpeakerMatch } from "./speakers";
|
||||
import { postAmbientCard } from "../../ui/shell/ambient-cards";
|
||||
import { pcmLevel, rmsToLevel } from "../../util/audio-level";
|
||||
import { formatRelativeTime } from "../../util/date-util";
|
||||
import { deviceLanguage, identifyLanguage, translateText } from "./translate";
|
||||
|
||||
@@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig {
|
||||
return { ...config, channelMask: 0b11 };
|
||||
}
|
||||
|
||||
function pcmLevel(pcm: Uint8Array): number {
|
||||
let sumSquares = 0;
|
||||
const count = pcm.length / 2;
|
||||
if (count === 0) return 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
if (value >= 0x8000) value -= 0x10000;
|
||||
sumSquares += value * value;
|
||||
}
|
||||
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
|
||||
}
|
||||
|
||||
function floatLevel(samples: Float32Array): number {
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
|
||||
return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length)));
|
||||
}
|
||||
|
||||
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
|
||||
function rmsToLevel(rms: number): number {
|
||||
if (rms <= 0) return 0;
|
||||
const db = 20 * Math.log10(rms);
|
||||
return Math.max(0, Math.min(1, (db + 60) / 60));
|
||||
}
|
||||
|
||||
function floatToPcm16(samples: Float32Array): Uint8Array {
|
||||
const out = new Uint8Array(samples.length * 2);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { NameInference } from "./name-guess";
|
||||
|
||||
/**
|
||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import { speakerRegistry } from "./speakers";
|
||||
|
||||
declare const com: any;
|
||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
+53
-14
@@ -8,8 +8,10 @@ import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { mediaControllerBridge, type MediaControllerState, type MediaQueueItem } from "../../native/media-controller";
|
||||
import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser";
|
||||
import { spotifyService } from "../../native/spotify";
|
||||
import { MediaBrowseLayer } from "./media-browse";
|
||||
import { MusicSettingsLayer } from "./music-settings";
|
||||
import { spotifyMenuItem } from "./spotify-menu";
|
||||
import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
|
||||
@@ -33,6 +35,8 @@ const QUEUE_X = 204;
|
||||
/** Horizontal inset of row text from its selection box. */
|
||||
const LIST_TEXT_INSET = 6;
|
||||
const PLAYLIST_ACTION_INDEX = 1;
|
||||
/** Minimum room for the dim artist after a queue row's title; less and it is left off. */
|
||||
const QUEUE_SUBTITLE_MIN_WIDTH = 40;
|
||||
|
||||
type MusicAction =
|
||||
| { kind: "play-pause"; label: string; enabled: boolean }
|
||||
@@ -40,14 +44,19 @@ type MusicAction =
|
||||
| { kind: "next"; label: string; enabled: boolean }
|
||||
| { kind: "volume"; label: string; enabled: boolean }
|
||||
| { kind: "playlist"; label: string; enabled: boolean }
|
||||
| { kind: "browse"; label: string; enabled: boolean };
|
||||
| { kind: "browse"; label: string; enabled: boolean }
|
||||
/** A player-specific action from the state's customActions (Spotify's shuffle / repeat / like). */
|
||||
| { kind: "custom"; label: string; enabled: boolean; action: string };
|
||||
|
||||
type FocusColumn = "actions" | "playlist";
|
||||
|
||||
/**
|
||||
* Music controller app: metadata + album art + transport controls for the
|
||||
* active Android media session (any player that publishes one) or, on iOS,
|
||||
* the Apple Music system player, plus the player's queue when it exposes one
|
||||
* the Apple Music system player; on both, Spotify through its Web API once
|
||||
* signed in from the window menu (optional on Android, where it adds
|
||||
* library browsing, shuffle / repeat / like and Spotify on other devices),
|
||||
* plus the player's queue when it exposes one
|
||||
* (scroll to a track, click to jump). With no session, the front page lists
|
||||
* the apps whose library can be browsed (click one to open its
|
||||
* MediaBrowseLayer).
|
||||
@@ -110,7 +119,7 @@ class MusicAppLayer implements Layer {
|
||||
const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh);
|
||||
this.libraryAppsFresh = true;
|
||||
this.libraryMenu.setItems(apps);
|
||||
image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180);
|
||||
image.drawText(font, 24, 16, global.isIOS ? media.status || "Nothing is playing." : "No active media session.", 180);
|
||||
const prompt = global.isIOS
|
||||
? "Browse your library, or start playback on the phone."
|
||||
: apps.length
|
||||
@@ -121,6 +130,11 @@ class MusicAppLayer implements Layer {
|
||||
image.drawText(font, 24, y, line, 150);
|
||||
y += lineStep(font);
|
||||
}
|
||||
// Why the last command failed (e.g. playing from the Spotify library with no device open).
|
||||
for (const line of media.notice ? wrapText(font, media.notice, width - 48) : []) {
|
||||
image.drawText(font, 24, y, line, 200);
|
||||
y += lineStep(font);
|
||||
}
|
||||
if (apps.length) {
|
||||
const listTop = y + 6;
|
||||
const listHeight = height - listTop - LIST_BOTTOM_MARGIN;
|
||||
@@ -156,12 +170,13 @@ class MusicAppLayer implements Layer {
|
||||
if (media.album) {
|
||||
image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150);
|
||||
}
|
||||
// The player line, or for a few seconds why a command failed.
|
||||
image.drawText(
|
||||
font,
|
||||
META_X,
|
||||
metaTop + 4 + (titleSlots + 2) * metaStep,
|
||||
truncateText(font, media.appName || media.packageName, metaWidth),
|
||||
110,
|
||||
truncateText(font, media.notice || media.appName || media.packageName, metaWidth),
|
||||
media.notice ? 200 : 110,
|
||||
);
|
||||
this.drawProgress(image, media, metaWidth);
|
||||
|
||||
@@ -212,7 +227,16 @@ class MusicAppLayer implements Layer {
|
||||
private drawQueueItem({ image, item, x, y, width, height, selected }: MenuDrawArgs<MediaQueueItem>): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const label = `${item.active ? "> " : " "}${item.title || "(untitled)"}`;
|
||||
image.drawText(font, x + LIST_TEXT_INSET, centeredTextY(font, y, height), truncateText(font, label, width - 2 * LIST_TEXT_INSET), selected ? 255 : 200);
|
||||
const textWidth = width - 2 * LIST_TEXT_INSET;
|
||||
const textY = centeredTextY(font, y, height);
|
||||
const title = truncateText(font, label, textWidth);
|
||||
image.drawText(font, x + LIST_TEXT_INSET, textY, title, selected ? 255 : 200);
|
||||
// The artist, dimmer, in whatever room the title leaves.
|
||||
const subtitleX = font.measureText(`${label} `);
|
||||
if (item.subtitle && title === label && textWidth - subtitleX >= QUEUE_SUBTITLE_MIN_WIDTH) {
|
||||
image.drawText(font, x + LIST_TEXT_INSET + subtitleX, textY,
|
||||
truncateText(font, item.subtitle, textWidth - subtitleX), selected ? 150 : 110);
|
||||
}
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
@@ -278,6 +302,7 @@ class MusicAppLayer implements Layer {
|
||||
else if (action.kind === "previous") await mediaControllerBridge.skipPrevious();
|
||||
else if (action.kind === "next") await mediaControllerBridge.skipNext();
|
||||
else if (action.kind === "browse") this.openBrowse(ctx);
|
||||
else if (action.kind === "custom") await mediaControllerBridge.sendCustomAction?.(action.action);
|
||||
else if (action.kind === "volume") {
|
||||
ctx.stack.push(new VolumeModalLayer(mediaControllerBridge.getMediaVolumePercent()));
|
||||
} else if (action.kind === "playlist") {
|
||||
@@ -342,6 +367,12 @@ class MusicAppLayer implements Layer {
|
||||
{ kind: "volume", label: volume >= 0 ? `Volume (${volume})` : "Volume", enabled: volume >= 0 },
|
||||
{ kind: "next", label: "Next track", enabled: media.canSkipNext },
|
||||
{ kind: "previous", label: "Previous track", enabled: media.canSkipPrevious },
|
||||
...(mediaControllerBridge.sendCustomAction ? media.customActions ?? [] : []).map((custom): MusicAction => ({
|
||||
kind: "custom",
|
||||
label: custom.name,
|
||||
enabled: true,
|
||||
action: custom.action,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -496,14 +527,22 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin
|
||||
iconLetter: "M",
|
||||
icon: "music",
|
||||
closeable: true,
|
||||
// Music settings choose among Android media apps; iOS has only Apple Music.
|
||||
menuItems: () => global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}],
|
||||
// Music settings choose among Android media apps (iOS has Apple Music
|
||||
// alone); Spotify's sign-in is optional on Android, where its media
|
||||
// session works without it.
|
||||
menuItems: () => {
|
||||
const spotify = spotifyService();
|
||||
return [
|
||||
...(global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx: LayerContext) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}]),
|
||||
...(spotify ? [spotifyMenuItem(spotify)] : []),
|
||||
];
|
||||
},
|
||||
actions: options.actions,
|
||||
baseLayer: new MusicRootLayer(musicLayer),
|
||||
submitFrame: options.submitFrame,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { GrayImage } from "../../graphics/image";
|
||||
import { isValidSpotifyClientId, SPOTIFY_DASHBOARD_URL, SPOTIFY_REDIRECT_URI } from "../../native/spotify-auth";
|
||||
import type { SpotifyService } from "../../native/spotify-service";
|
||||
import { spotifyClientIdSetting, textSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { submenuItem, TextPageLayer, type MenuItem } from "../../ui/menu";
|
||||
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||
import { TextViewerLayer } from "../files/text-viewer";
|
||||
|
||||
const SETUP_INTRO = global.isIOS
|
||||
? "Faceclaw controls Spotify through Spotify's Web API, which needs a Spotify developer app of your own. Creating one is free; Spotify requires its owner to have Premium, and controlling playback needs Premium too."
|
||||
: "Faceclaw already shows and controls Spotify on this phone, like any music app. Signing in to Spotify's Web API is optional and adds browsing your Spotify library, playing from Spotify by voice, shuffle / repeat / like, and Spotify playing on your other devices. It needs a Spotify developer app of your own: creating one is free, Spotify requires its owner to have Premium, and controlling playback through it needs Premium too.";
|
||||
|
||||
const SETUP_HELP = [
|
||||
SETUP_INTRO,
|
||||
"",
|
||||
`1. On a computer, open ${SPOTIFY_DASHBOARD_URL.replace("https://", "")} and sign in with your Spotify account.`,
|
||||
"2. Create an app. Any name and description will do.",
|
||||
`3. Add this Redirect URI exactly: ${SPOTIFY_REDIRECT_URI}`,
|
||||
"4. Under \"Which API/SDKs are you planning to use?\", tick Web API, then save.",
|
||||
"5. Open the app's settings, copy its Client ID, and enter it as Client ID here.",
|
||||
"6. Choose Sign in, then finish on your phone: Faceclaw opens Spotify's sign-in page there (once Faceclaw is open on the phone, if it was in the background).",
|
||||
"",
|
||||
"Spotify plays on whichever of your devices is open (this phone, a computer, a speaker). To start music when nothing is playing, Spotify must be open on one of them.",
|
||||
"",
|
||||
"Others can sign in with your client ID once you add their Spotify email under User Management in the dashboard (up to five people).",
|
||||
].join("\n");
|
||||
|
||||
/** The Music app menu's Spotify row: sign in, sign out, client ID, setup help. */
|
||||
export function spotifyMenuItem(service: SpotifyService): MenuItem {
|
||||
return submenuItem("Spotify", (ctx) => ctx.stack.push(new SpotifyMenuLayer(service)), {
|
||||
description: global.isIOS
|
||||
? "Sign in to Spotify to show and control it from the glasses."
|
||||
: "Optional: sign in to Spotify to browse its library and play from it by voice.",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows rebuilt every paint and input from the account and sign-in progress,
|
||||
* which change asynchronously. A WindowMenuLayer, so the shell's system menu
|
||||
* closes it like the window menu it opens from.
|
||||
*/
|
||||
class SpotifyMenuLayer extends WindowMenuLayer {
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
constructor(private readonly service: SpotifyService) {
|
||||
super("Spotify", []);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
this.unsubscribe ??= this.service.onChange(() => ctx.actions.requestRender());
|
||||
this.setItems(this.rows());
|
||||
return super.paint(ctx, paintBelow);
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
this.setItems(this.rows());
|
||||
await super.handleInput(event, ctx);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
private rows(): MenuItem[] {
|
||||
const service = this.service;
|
||||
const account = service.account();
|
||||
const status = service.signInStatus();
|
||||
const help: MenuItem = {
|
||||
label: "Setup help",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextViewerLayer(SETUP_HELP, "Spotify setup")),
|
||||
};
|
||||
if (account) {
|
||||
return [
|
||||
info(`Signed in as ${account.displayName || account.userId}`),
|
||||
...(account.product && account.product !== "premium" ? [info("Playback control needs Premium")] : []),
|
||||
{ label: "Sign out", onSelect: () => service.signOut() },
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
];
|
||||
}
|
||||
if (status.phase === "waiting") {
|
||||
return [
|
||||
info("Finish signing in on the phone"),
|
||||
{ label: "Cancel sign-in", onSelect: () => service.cancelSignIn() },
|
||||
help,
|
||||
];
|
||||
}
|
||||
const failure: MenuItem[] = status.error
|
||||
? [{
|
||||
label: "Sign-in failed (details)",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextPageLayer("Spotify sign-in failed", status.error)),
|
||||
}]
|
||||
: [];
|
||||
return [
|
||||
{
|
||||
label: "Sign in",
|
||||
description: "Opens Spotify's sign-in page on the phone.",
|
||||
disabled: () => !isValidSpotifyClientId(spotifyClientIdSetting.get()),
|
||||
onSelect: () => void service.signIn(),
|
||||
},
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
...failure,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
function info(label: string): MenuItem {
|
||||
return { label, disabled: true, onSelect: () => {} };
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import { WindowMenu, WindowMenuLayer } from "../../ui/window-menu";
|
||||
import type { LayerContext } from "../../ui/layers";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { onSettingsStoreChanged } from "../../native/settings-store";
|
||||
import { formatClockTime } from "../../ui/clock-format";
|
||||
import { openUrlOnPhone } from "../../native/open-url";
|
||||
import {
|
||||
navigateDestinationAddressDraftSetting,
|
||||
@@ -35,6 +36,9 @@ import {
|
||||
navigateRememberRecentSetting,
|
||||
navigateDisplayLayoutSetting,
|
||||
mapboxApiKeySetting,
|
||||
distanceUnitSetting,
|
||||
DISTANCE_UNIT_SETTING_KEY,
|
||||
TIME_FORMAT_SETTING_KEY,
|
||||
enumSettingMenuItem,
|
||||
type ConfigSettingString,
|
||||
} from "../../ui/dashboard-settings";
|
||||
@@ -266,8 +270,15 @@ function post(message: WorkerAppReply): void {
|
||||
// the edit screen live.
|
||||
onSettingsStoreChanged((key) => {
|
||||
// The token key repaints too: the phone editor writes it live during
|
||||
// Edit token, and the idle page changes shape once one is set.
|
||||
if (key.startsWith("navigate.") || key === MAPBOX_TOKEN_SETTING_KEY) render();
|
||||
// Edit token, and the idle page changes shape once one is set. So do the
|
||||
// distance unit and time format (Settings > Customization), for the
|
||||
// guidance panel.
|
||||
if (
|
||||
key.startsWith("navigate.") ||
|
||||
key === MAPBOX_TOKEN_SETTING_KEY ||
|
||||
key === DISTANCE_UNIT_SETTING_KEY ||
|
||||
key === TIME_FORMAT_SETTING_KEY
|
||||
) render();
|
||||
});
|
||||
|
||||
// The host queues messages until this arrives: posts to a worker whose bundle
|
||||
@@ -1569,15 +1580,33 @@ function drawFooter(image: GrayImage, text: string, x = 16): void {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Formatting (imperial; matches the rest of the UI)
|
||||
// Formatting (distances and clock times follow Settings > Customization)
|
||||
|
||||
function metricDistances(): boolean {
|
||||
return distanceUnitSetting.get() === "kilometers";
|
||||
}
|
||||
|
||||
/** Under a kilometer: 10 m steps up to 100 m, then 50 m steps. */
|
||||
function formatMeters(meters: number): string {
|
||||
const step = meters < 100 ? 10 : 50;
|
||||
return `${Math.max(0, Math.round(meters / step) * step)} m`;
|
||||
}
|
||||
|
||||
function formatDistanceShort(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
return meters < 950 ? formatMeters(meters) : `${(meters / 1000).toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 900) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
return `${(feet / 5280).toFixed(1)} mi`;
|
||||
}
|
||||
|
||||
function formatDistance(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
if (meters < 950) return formatMeters(meters);
|
||||
const km = meters / 1000;
|
||||
return km >= 10 ? `${Math.round(km)} km` : `${km.toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 1000) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
const miles = feet / 5280;
|
||||
@@ -1597,14 +1626,9 @@ function formatDurationLong(seconds: number): string {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
}
|
||||
|
||||
/** "14:05" / "2:05 PM", as the top-bar clock shows it. */
|
||||
function formatClock(timestampMs: number): string {
|
||||
const date = new Date(timestampMs);
|
||||
let hours = date.getHours();
|
||||
const minutes = String(date.getMinutes()).padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
return formatClockTime(new Date(timestampMs));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer {
|
||||
private detail = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private readonly meter = new MicLevelMeter();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
@@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer {
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
@@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer {
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
@@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { TranslateLayer } from "./translate";
|
||||
import { resolveTranslationModel } from "./translate-llm";
|
||||
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
|
||||
import {
|
||||
resolveTranslateLanguage,
|
||||
resolveTranslateProvider,
|
||||
translateLanguageSetting,
|
||||
translateProviderSetting,
|
||||
translateShowOriginalSetting,
|
||||
} from "./translate-settings";
|
||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
@@ -22,8 +28,8 @@ const TRAY_ICON_ID = "translate";
|
||||
const NO_PROVIDER_PROBLEM =
|
||||
"Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice).";
|
||||
const NO_MODEL_PROBLEM = global.isIOS
|
||||
? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)."
|
||||
: "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant).";
|
||||
? "Translate needs a language model: configure a cloud or OpenAI-compatible model in Settings > Assistant."
|
||||
: "Translate needs a language model: configure one in Settings > Assistant, or download the on-phone model.";
|
||||
|
||||
export type TranslateAppOptions = InProcessAppOptions & {
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
@@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & {
|
||||
* provider (Translate settings), which a shared capture switches to.
|
||||
*/
|
||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||
// Checks the setup and (re)starts capture with the chosen provider: at
|
||||
// launch, when the provider setting changes, and on a tap after a problem.
|
||||
// Checks the setup and (re)starts capture with the chosen provider and
|
||||
// language: at launch, when either setting changes, and on a tap after a
|
||||
// problem.
|
||||
const startCapture = () => {
|
||||
const provider = resolveTranslateProvider();
|
||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||
@@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
return;
|
||||
}
|
||||
options.startContinuousVoiceCapture({ multilingualProvider: provider });
|
||||
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
};
|
||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
||||
@@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Translate settings", [
|
||||
enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }),
|
||||
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
|
||||
toggleSettingMenuItem(translateShowOriginalSetting),
|
||||
]);
|
||||
@@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
||||
startCapture();
|
||||
return app;
|
||||
}
|
||||
@@ -4,11 +4,15 @@
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import {
|
||||
anthropicApiKeySetting, assistantModelSetting, assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts";
|
||||
import { type TranslationTurn } from "./translation-model";
|
||||
|
||||
@@ -23,12 +27,17 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null {
|
||||
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
}
|
||||
|
||||
export type TranslationRequest = {
|
||||
speech: string;
|
||||
context: TranslationTurn[];
|
||||
/** English name of the spoken language when the user set one, else "". */
|
||||
language: string;
|
||||
onProgress: (textSoFar: string) => void;
|
||||
onDone: (translation: string) => void;
|
||||
onError: (message: string) => void;
|
||||
@@ -41,7 +50,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
system: TRANSLATE_SYSTEM_PROMPT,
|
||||
effort: model.effort,
|
||||
maxTokens: 1024,
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }],
|
||||
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
|
||||
onDone: (result) => {
|
||||
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
|
||||
@@ -52,8 +61,10 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
switch (model.provider) {
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "openai-compatible":
|
||||
return streamOpenAiCompatibleChat({ ...options, baseUrl: model.baseUrl ?? "" });
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
return streamLocalModel(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
|
||||
@@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>
|
||||
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Spoken languages Translate can be told to expect, by ISO 639-1 code (the
|
||||
* form every multilingual provider accepts), in picker order.
|
||||
*/
|
||||
const SPOKEN_LANGUAGES = {
|
||||
ar: "Arabic",
|
||||
zh: "Chinese",
|
||||
nl: "Dutch",
|
||||
fr: "French",
|
||||
de: "German",
|
||||
el: "Greek",
|
||||
he: "Hebrew",
|
||||
hi: "Hindi",
|
||||
id: "Indonesian",
|
||||
it: "Italian",
|
||||
ja: "Japanese",
|
||||
ko: "Korean",
|
||||
pl: "Polish",
|
||||
pt: "Portuguese",
|
||||
ru: "Russian",
|
||||
es: "Spanish",
|
||||
sv: "Swedish",
|
||||
th: "Thai",
|
||||
tr: "Turkish",
|
||||
uk: "Ukrainian",
|
||||
vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES;
|
||||
|
||||
export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({
|
||||
id: "translate-language",
|
||||
label: "Spoken language",
|
||||
storageKey: "translate.language",
|
||||
defaultValue: "auto",
|
||||
values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])],
|
||||
formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]),
|
||||
description:
|
||||
"The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.",
|
||||
});
|
||||
|
||||
/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */
|
||||
export function resolveTranslateLanguage(): SpokenLanguage | "" {
|
||||
const value = translateLanguageSetting.get();
|
||||
return value === "auto" ? "" : value;
|
||||
}
|
||||
|
||||
/** The English name of a spoken language, for the translation prompt. */
|
||||
export function spokenLanguageName(language: SpokenLanguage): string {
|
||||
return SPOKEN_LANGUAGES[language];
|
||||
}
|
||||
|
||||
export const translateShowOriginalSetting = new ConfigSettingBoolean({
|
||||
id: "translate-show-original",
|
||||
label: "Show original",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
||||
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
||||
import { translateShowOriginalSetting } from "./translate-settings";
|
||||
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
||||
|
||||
const NO_MODEL_NOTICE = "No language model is set up.";
|
||||
|
||||
/**
|
||||
* Live translation view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranslateAppWindow); this layer feeds finalized speech to the
|
||||
* assistant's model one stretch at a time, shows the English as it streams
|
||||
* in, and saves the session to Downloads from the app menu.
|
||||
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
||||
* model one block at a time (provisionally while a block is still being
|
||||
* spoken), shows the English as it streams in, and saves the session to
|
||||
* Downloads from the app menu.
|
||||
*/
|
||||
export class TranslateLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -30,7 +32,8 @@ export class TranslateLayer implements Layer {
|
||||
private problem: string | null = null;
|
||||
private readonly model = new TranslationModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private activeJob: TranslationJob | null = null;
|
||||
private readonly meter = new MicLevelMeter();
|
||||
// The request in flight (the model tracks which job it is).
|
||||
private activeHandle: LlmStreamHandle | null = null;
|
||||
private requestRender: () => void = () => {};
|
||||
private removed = false;
|
||||
@@ -40,10 +43,16 @@ export class TranslateLayer implements Layer {
|
||||
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
||||
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.requestRender = requestRender;
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.model.accept(event);
|
||||
if (this.model.accept(event)) {
|
||||
// The request in flight translates text the transcript has replaced.
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
}
|
||||
this.pump();
|
||||
requestRender();
|
||||
});
|
||||
@@ -68,9 +77,9 @@ export class TranslateLayer implements Layer {
|
||||
const measure = (line: string) => font.measureText(line);
|
||||
const maxWidth = width - 64;
|
||||
|
||||
image.drawText(font, 24, 20, "Translate", 200);
|
||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
||||
image.drawText(font, 24, 40, this.saveNotice || status, 110);
|
||||
const meter = this.listening && !this.problem ? this.meter : null;
|
||||
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
|
||||
|
||||
let lines: TranslationLine[];
|
||||
if (this.problem) {
|
||||
@@ -84,7 +93,6 @@ export class TranslateLayer implements Layer {
|
||||
}
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
||||
@@ -119,7 +127,7 @@ export class TranslateLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.activeJob = null;
|
||||
this.meter.stop();
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
@@ -139,7 +147,10 @@ export class TranslateLayer implements Layer {
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
/** Start translating the next waiting stretch of speech, unless one is in flight. */
|
||||
/**
|
||||
* Start the next translation request (a final block, else the live block's
|
||||
* provisional one), unless one is in flight.
|
||||
*/
|
||||
private pump(): void {
|
||||
if (this.removed) return;
|
||||
const job = this.model.nextJob();
|
||||
@@ -147,36 +158,34 @@ export class TranslateLayer implements Layer {
|
||||
const llm = resolveTranslationModel();
|
||||
if (!llm) {
|
||||
// The model went away mid-session (a key was cleared): keep the original.
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = NO_MODEL_NOTICE;
|
||||
this.pump();
|
||||
return;
|
||||
}
|
||||
this.activeJob = job;
|
||||
const language = resolveTranslateLanguage();
|
||||
const handle = streamTranslation(llm, {
|
||||
speech: job.segment.source,
|
||||
speech: job.source,
|
||||
context: job.context,
|
||||
language: language ? spokenLanguageName(language) : "",
|
||||
onProgress: (textSoFar) => {
|
||||
if (this.activeJob !== job) return;
|
||||
this.model.progress(job.segment, textSoFar);
|
||||
this.requestRender();
|
||||
if (this.model.progress(job, textSoFar)) this.requestRender();
|
||||
},
|
||||
onDone: (translation) => this.settle(job, () => {
|
||||
this.model.complete(job.segment, translation);
|
||||
this.model.complete(job, translation);
|
||||
this.translationError = "";
|
||||
}),
|
||||
onError: (message) => this.settle(job, () => {
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = `Translation failed: ${message}`;
|
||||
}),
|
||||
});
|
||||
// A provider that fails synchronously has already settled (and moved on).
|
||||
if (this.activeJob === job) this.activeHandle = handle;
|
||||
if (this.model.isActive(job)) this.activeHandle = handle;
|
||||
}
|
||||
|
||||
private settle(job: TranslationJob, apply: () => void): void {
|
||||
if (this.activeJob !== job) return;
|
||||
this.activeJob = null;
|
||||
if (!this.model.isActive(job)) return;
|
||||
this.activeHandle = null;
|
||||
apply();
|
||||
this.pump();
|
||||
|
||||
@@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
|
||||
type TranscriptEvent = { text: string; isFinal: boolean };
|
||||
|
||||
export type TranslationState = "pending" | "translating" | "done" | "failed";
|
||||
|
||||
/** One stretch of finalized speech and its English translation. */
|
||||
/**
|
||||
* One block of speech (up to a pause) and its English translation. The last
|
||||
* block may still be live, its transcript revising; it is translated
|
||||
* provisionally as it grows, then again once final if its text changed.
|
||||
*/
|
||||
export type TranslationSegment = {
|
||||
source: string;
|
||||
/** The translation so far (streamed while translating). */
|
||||
/** Committed by the transcriber; a live block's source still revises. */
|
||||
final: boolean;
|
||||
/** The latest finished translation, of `translatedSource`. */
|
||||
translation: string;
|
||||
state: TranslationState;
|
||||
/** What `translation` translates: null until one finishes, behind `source` while provisional. */
|
||||
translatedSource: string | null;
|
||||
/** The text streamed so far by the request in flight for this block, else null. */
|
||||
streaming: string | null;
|
||||
/** A source whose translation failed; not retried until the source changes. */
|
||||
failedSource: string | null;
|
||||
};
|
||||
|
||||
export type TranslationTurn = { source: string; translation: string };
|
||||
|
||||
/** A segment handed to the model, with the finished turns before it. */
|
||||
/** A request: a block's source as sent, with the finished turns before it. */
|
||||
export type TranslationJob = {
|
||||
segment: TranslationSegment;
|
||||
/** The source sent (a live block's moves on while the request is in flight). */
|
||||
source: string;
|
||||
/** Earlier translated turns, oldest first. */
|
||||
context: TranslationTurn[];
|
||||
};
|
||||
@@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number };
|
||||
// over between requests; bounded so a long session's requests stay small.
|
||||
const CONTEXT_TURNS = 4;
|
||||
const CONTEXT_CHARS = 800;
|
||||
// A live block is translated again once its text has changed length by this
|
||||
// much since the last request, so a trickle of revisions (punctuation, one
|
||||
// corrected word) doesn't spend a request each. A final is always redone.
|
||||
const PROVISIONAL_MIN_CHANGE = 3;
|
||||
|
||||
const TRANSLATION_GRAY = 230;
|
||||
const PROVISIONAL_GRAY = 170;
|
||||
const ORIGINAL_GRAY = 120;
|
||||
const LIVE_GRAY = 90;
|
||||
|
||||
function newSegment(source: string, final: boolean): TranslationSegment {
|
||||
return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null };
|
||||
}
|
||||
|
||||
/** Whether a block's current source still needs a translation request. */
|
||||
function wantsTranslation(segment: TranslationSegment): boolean {
|
||||
const { source, translatedSource } = segment;
|
||||
if (!source || source === translatedSource || source === segment.failedSource) return false;
|
||||
return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Translate app's conversation: finalized speech segments with their
|
||||
* translations, plus the live partial of the utterance in progress.
|
||||
* The Translate app's conversation: blocks of speech with their translations.
|
||||
* Translation is serial (one request at a time, in order, so each can carry
|
||||
* the turns before it as context); finals that arrive while a request is in
|
||||
* flight merge into one pending segment, so a slow model batches the backlog
|
||||
* instead of falling further behind one request per pause.
|
||||
* the turns before it as context): final blocks first, then the live one.
|
||||
* Finals that arrive while a request is in flight merge into one waiting
|
||||
* block, so a slow model batches the backlog instead of falling further
|
||||
* behind one request per pause.
|
||||
*/
|
||||
export class TranslationModel {
|
||||
readonly segments: TranslationSegment[] = [];
|
||||
/** The utterance in progress (replace semantics), not yet final. */
|
||||
live = "";
|
||||
private active: TranslationJob | null = null;
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
/**
|
||||
* Take a transcript event (replace semantics within the live block).
|
||||
* Returns true when it made the request in flight moot (its block was
|
||||
* finalized as other text, or dropped); that request is abandoned here, and
|
||||
* the caller should cancel it.
|
||||
*/
|
||||
accept(event: TranscriptEvent): boolean {
|
||||
const text = event.text.trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
this.live = "";
|
||||
if (!finalText) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
|
||||
else this.segments.push({ source: finalText, translation: "", state: "pending" });
|
||||
const live = last && !last.final ? last : null;
|
||||
if (!event.isFinal) {
|
||||
if (live) live.source = text;
|
||||
else if (text) this.segments.push(newSegment(text, false));
|
||||
return false;
|
||||
}
|
||||
const finalText = text || live?.source || "";
|
||||
if (!live) {
|
||||
if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true));
|
||||
return false;
|
||||
}
|
||||
const moot = this.active?.segment === live && this.active.source !== finalText;
|
||||
if (moot) this.abandonActive();
|
||||
if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) {
|
||||
this.segments.pop();
|
||||
return moot;
|
||||
}
|
||||
live.source = finalText;
|
||||
live.final = true;
|
||||
return moot;
|
||||
}
|
||||
|
||||
/** Take the next segment to translate, or null while one is in flight or none waits. */
|
||||
/** Take the next request to make, or null while one is in flight or nothing waits. */
|
||||
nextJob(): TranslationJob | null {
|
||||
if (this.segments.some((segment) => segment.state === "translating")) return null;
|
||||
const segment = this.segments.find((candidate) => candidate.state === "pending");
|
||||
if (this.active) return null;
|
||||
const segment = this.segments.find((candidate) => wantsTranslation(candidate));
|
||||
if (!segment) return null;
|
||||
segment.state = "translating";
|
||||
return { segment, context: this.contextBefore(segment) };
|
||||
segment.streaming = "";
|
||||
this.active = { segment, source: segment.source, context: this.contextBefore(segment) };
|
||||
return this.active;
|
||||
}
|
||||
|
||||
progress(segment: TranslationSegment, textSoFar: string): void {
|
||||
if (segment.state === "translating") segment.translation = textSoFar;
|
||||
isActive(job: TranslationJob): boolean {
|
||||
return this.active === job;
|
||||
}
|
||||
|
||||
complete(segment: TranslationSegment, translation: string): void {
|
||||
segment.translation = translation;
|
||||
segment.state = "done";
|
||||
/** Streamed text for the request in flight; false when `job` isn't it any more. */
|
||||
progress(job: TranslationJob, textSoFar: string): boolean {
|
||||
if (this.active !== job) return false;
|
||||
job.segment.streaming = textSoFar;
|
||||
return true;
|
||||
}
|
||||
|
||||
fail(segment: TranslationSegment): void {
|
||||
segment.translation = "";
|
||||
segment.state = "failed";
|
||||
complete(job: TranslationJob, translation: string): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.translation = translation;
|
||||
job.segment.translatedSource = job.source;
|
||||
}
|
||||
|
||||
fail(job: TranslationJob): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.failedSource = job.source;
|
||||
}
|
||||
|
||||
private abandonActive(): void {
|
||||
if (this.active) this.active.segment.streaming = null;
|
||||
this.active = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a final to the block before `before` (default: the end) when that
|
||||
* block is final but still waiting for its first request. Returns whether
|
||||
* it did.
|
||||
*/
|
||||
private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean {
|
||||
const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1;
|
||||
const waiting = this.segments[index];
|
||||
if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false;
|
||||
if (this.active?.segment === waiting) return false;
|
||||
waiting.source = `${waiting.source} ${finalText}`;
|
||||
return true;
|
||||
}
|
||||
|
||||
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
|
||||
@@ -87,7 +162,7 @@ export class TranslationModel {
|
||||
let chars = 0;
|
||||
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
|
||||
const earlier = this.segments[index]!;
|
||||
if (earlier.state !== "done" || !earlier.translation) continue;
|
||||
if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue;
|
||||
chars += earlier.source.length + earlier.translation.length;
|
||||
if (turns.length && chars > CONTEXT_CHARS) break;
|
||||
turns.unshift({ source: earlier.source, translation: earlier.translation });
|
||||
@@ -96,9 +171,11 @@ export class TranslationModel {
|
||||
}
|
||||
|
||||
/**
|
||||
* Display lines: each segment's translation, bright; its original, dim,
|
||||
* above it when `showOriginal` is on and in its place until a translation
|
||||
* arrives (or when it failed); then the live partial, dimmer.
|
||||
* Display lines: each block's translation, bright once settled (final, and
|
||||
* of its final text) and dimmer while provisional; its original, dim, above
|
||||
* it when `showOriginal` is on and in its place until a translation
|
||||
* arrives. A redo streams in over the translation it replaces only once it
|
||||
* is as long, so the text doesn't shrink and regrow.
|
||||
*/
|
||||
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
|
||||
const lines: TranslationLine[] = [];
|
||||
@@ -106,21 +183,25 @@ export class TranslationModel {
|
||||
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
|
||||
};
|
||||
for (const segment of this.segments) {
|
||||
const translation = segment.translation.trim();
|
||||
// A finished, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
|
||||
if (translation) push(translation, TRANSLATION_GRAY);
|
||||
if (!segment.source) continue;
|
||||
const settled = segment.final && segment.translatedSource === segment.source;
|
||||
const finished = segment.translation.trim();
|
||||
const streaming = segment.streaming?.trim() ?? "";
|
||||
const translation = streaming.length >= finished.length ? streaming : finished;
|
||||
// A settled, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY);
|
||||
if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY);
|
||||
}
|
||||
if (this.live) push(this.live, LIVE_GRAY);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Plain-text transcript for saving: each original, then its translation. */
|
||||
/** Plain-text transcript for saving: each final original, then its translation. */
|
||||
savedText(): string {
|
||||
return this.segments
|
||||
.filter((segment) => segment.final)
|
||||
.map((segment) => {
|
||||
const translation = segment.translation.trim();
|
||||
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation || translation === segment.source.trim()) return segment.source;
|
||||
return `${segment.source}\n→ ${translation}`;
|
||||
})
|
||||
|
||||
@@ -6,6 +6,7 @@ import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { distanceUnitSetting } from "../../ui/dashboard-settings";
|
||||
import { renderWeatherIcon } from "./weather-icons";
|
||||
import { formatTemperature, widestTemperatureSample } from "./temperature-format";
|
||||
|
||||
@@ -142,7 +143,7 @@ export class WeatherLayer implements Layer {
|
||||
const details: string[] = [];
|
||||
if (current.humidityPercent !== null) details.push(`Humidity ${Math.round(current.humidityPercent)}%`);
|
||||
if (current.windSpeedMph !== null) {
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${Math.round(current.windSpeedMph)} mph`);
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${formatWindSpeed(current.windSpeedMph)}`);
|
||||
}
|
||||
image.drawText(small, textX, cy, details.join(" ") || "Current forecast", 170);
|
||||
cy += lineStep(small);
|
||||
@@ -247,6 +248,11 @@ function forecastRowHeight(font: UiFont): number {
|
||||
return Math.max(FORECAST_MIN_ROW_HEIGHT, tightRowHeight(font) + 3);
|
||||
}
|
||||
|
||||
/** A wind speed (weather data is kept in mph) in the Distance unit setting's speed unit: "12 mph" / "19 km/h". */
|
||||
function formatWindSpeed(mph: number): string {
|
||||
return distanceUnitSetting.get() === "kilometers" ? `${Math.round(mph * 1.609344)} km/h` : `${Math.round(mph)} mph`;
|
||||
}
|
||||
|
||||
function formatAge(ageMs: number): string {
|
||||
const minutes = Math.max(0, Math.round(ageMs / 60_000));
|
||||
if (minutes < 1) return "just now";
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamHermesChat } from "../native/hermes";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamLocalModel } from "../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../native/openai-chat";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
@@ -40,6 +41,8 @@ export type DirectTurnOptions = {
|
||||
effort?: "low" | "medium" | "high";
|
||||
/** Hermes: the API root, and the server-side session this conversation maps to. */
|
||||
baseUrl?: string;
|
||||
/** Venice-only web search for this interactive turn. */
|
||||
webSearch?: boolean;
|
||||
sessionId?: string;
|
||||
system: string;
|
||||
/** Owned by the session; this loop appends assistant + tool_result messages. */
|
||||
@@ -85,11 +88,12 @@ export class DirectAssistantBackend {
|
||||
return;
|
||||
}
|
||||
|
||||
const streamOptions: LlmStreamOptions = {
|
||||
const streamOptions: LlmStreamOptions & { webSearch?: boolean } = {
|
||||
apiKey: options.apiKey,
|
||||
model: options.model,
|
||||
system: options.system,
|
||||
effort: options.effort,
|
||||
webSearch: options.webSearch,
|
||||
messages: options.messages,
|
||||
// Hermes runs its own tools; Faceclaw's are not offered to it.
|
||||
tools: options.provider === "hermes" ? [] : options.buildTools(),
|
||||
@@ -143,10 +147,12 @@ export class DirectAssistantBackend {
|
||||
streamHandle =
|
||||
options.provider === "hermes"
|
||||
? streamHermesChat({ ...streamOptions, baseUrl: options.baseUrl ?? "", sessionId: options.sessionId ?? "" })
|
||||
: options.provider === "openai-compatible"
|
||||
? streamOpenAiCompatibleChat({ ...streamOptions, baseUrl: options.baseUrl ?? "" })
|
||||
: options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
? streamLocalModel(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ export type LlmContentBlock =
|
||||
}
|
||||
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }
|
||||
/** Opaque state required to continue a provider's stateless conversation. */
|
||||
| { type: "provider_item"; provider: "openai"; item: any };
|
||||
| { type: "provider_item"; provider: "openai" | "openai-compatible"; item: any };
|
||||
|
||||
export type LlmMessage = {
|
||||
role: "user" | "assistant";
|
||||
|
||||
+38
-9
@@ -1,4 +1,5 @@
|
||||
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
|
||||
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
|
||||
import { isVeniceApiBaseUrl } from "../native/openai-chat";
|
||||
|
||||
export const ASSISTANT_MODEL_VALUES = [
|
||||
"auto",
|
||||
@@ -11,17 +12,23 @@ export const ASSISTANT_MODEL_VALUES = [
|
||||
"sol",
|
||||
"qwen",
|
||||
"hermes",
|
||||
"openai-compatible",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
|
||||
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
|
||||
* support and stays for saved settings and conversations).
|
||||
*/
|
||||
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
||||
/** Keep saved model IDs portable, while offering only supported choices. */
|
||||
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
||||
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
|
||||
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
|
||||
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
||||
return global.isIOS && value === "qwen" ? "auto" : value;
|
||||
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
|
||||
}
|
||||
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes" | "openai-compatible";
|
||||
|
||||
export type AssistantApiKeys = {
|
||||
anthropic: string;
|
||||
@@ -29,6 +36,11 @@ export type AssistantApiKeys = {
|
||||
/** A Hermes Agent API server; only the voice assistant passes these. */
|
||||
hermesUrl?: string;
|
||||
hermesKey?: string;
|
||||
openAiCompatibleUrl?: string;
|
||||
openAiCompatibleKey?: string;
|
||||
openAiCompatibleModel?: string;
|
||||
/** Venice web search, applied only when the resolved root is Venice. */
|
||||
veniceWebSearch?: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedAssistantModel = {
|
||||
@@ -37,8 +49,10 @@ export type ResolvedAssistantModel = {
|
||||
model: string;
|
||||
apiKey: string;
|
||||
effort?: "low" | "medium" | "high";
|
||||
/** API root for providers that are not fixed services (Hermes). */
|
||||
/** API root for Hermes or an OpenAI-compatible provider. */
|
||||
baseUrl?: string;
|
||||
/** Whether Venice web search is enabled for this interactive assistant turn. */
|
||||
webSearch?: boolean;
|
||||
};
|
||||
|
||||
type ModelDefinition = {
|
||||
@@ -58,11 +72,13 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
|
||||
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
||||
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
||||
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
||||
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
|
||||
// not an API key. The default for users who haven't set any key.
|
||||
// Runs on the phone itself (Android: llama.cpp with the downloaded model
|
||||
// file; iOS: Apple Intelligence), not an API key. The default for users who
|
||||
// haven't set any key.
|
||||
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
||||
// Your own Hermes Agent (its API server), with its own tools and memory.
|
||||
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
||||
"openai-compatible": { label: "OpenAI-compatible", provider: "openai-compatible", model: "custom" },
|
||||
};
|
||||
|
||||
export function assistantModelLabel(value: AssistantModel): string {
|
||||
@@ -76,8 +92,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
|
||||
/**
|
||||
* Resolve Auto and attach the credential needed by the chosen provider.
|
||||
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
||||
* and falls back to the on-phone model when no key is set but the model file
|
||||
* has been downloaded.
|
||||
* and falls back to the on-phone model when no key is set but that model is
|
||||
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
|
||||
*/
|
||||
export function resolveAssistantModel(
|
||||
selection: AssistantModel,
|
||||
@@ -106,6 +122,19 @@ export function resolveAssistantModel(
|
||||
if (!baseUrl) return null;
|
||||
return { selection, provider: "hermes", model: definition.model, apiKey: keys.hermesKey?.trim() ?? "", baseUrl };
|
||||
}
|
||||
if (definition.provider === "openai-compatible") {
|
||||
const baseUrl = keys.openAiCompatibleUrl?.trim() ?? "";
|
||||
const model = keys.openAiCompatibleModel?.trim() ?? "";
|
||||
if (!baseUrl || !model) return null;
|
||||
return {
|
||||
selection,
|
||||
provider: definition.provider,
|
||||
model,
|
||||
apiKey: keys.openAiCompatibleKey?.trim() ?? "",
|
||||
baseUrl,
|
||||
webSearch: isVeniceApiBaseUrl(baseUrl) && keys.veniceWebSearch === true,
|
||||
};
|
||||
}
|
||||
if (definition.provider === "local") {
|
||||
if (!isLocalModelReady()) return null;
|
||||
return {
|
||||
|
||||
@@ -95,14 +95,20 @@ export class AssistantSession {
|
||||
// original model. Preserve visible conversation when switching engines.
|
||||
if (this.config.kind !== config.kind ||
|
||||
(this.config.kind === "direct" && config.kind === "direct" &&
|
||||
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
|
||||
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider ||
|
||||
(this.config.llm.baseUrl ?? "") !== (config.llm.baseUrl ?? "")))) {
|
||||
this.rebuildTextHistory();
|
||||
}
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private engine(): string {
|
||||
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
|
||||
if (this.config.kind !== "direct") return "external";
|
||||
const llm = this.config.llm;
|
||||
// Keep the legacy engine ID for built-in providers so upgrading does not
|
||||
// discard their opaque history. Custom endpoints need endpoint isolation.
|
||||
const endpoint = llm.provider === "openai-compatible" ? `:${llm.baseUrl ?? ""}` : "";
|
||||
return `${llm.provider}:${llm.model}${endpoint}`;
|
||||
}
|
||||
|
||||
private rebuildTextHistory(): void {
|
||||
@@ -192,6 +198,7 @@ export class AssistantSession {
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
baseUrl: llm.baseUrl,
|
||||
webSearch: llm.webSearch,
|
||||
sessionId: this.hermesSessionId,
|
||||
system: llm.provider === "hermes" ? HERMES_SYSTEM_PROMPT
|
||||
: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { readUpcomingEventsAsync, type CalendarEvent } from "../native/calendar";
|
||||
import { hasCalendarPermission } from "../native/calendar-permissions";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { spotifyService } from "../native/spotify";
|
||||
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
|
||||
@@ -93,7 +94,11 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeNowPlaying()),
|
||||
async () => {
|
||||
// Spotify's state is polled only while something shows it.
|
||||
await mediaControllerBridge.refresh?.();
|
||||
return ok(describeNowPlaying());
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
@@ -103,6 +108,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
await mediaControllerBridge.refresh?.();
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canPlayPause) return err("No media session can be controlled right now.");
|
||||
await mediaControllerBridge.playPause();
|
||||
@@ -117,6 +123,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
await mediaControllerBridge.refresh?.();
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
|
||||
await mediaControllerBridge.skipNext();
|
||||
@@ -124,6 +131,45 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
},
|
||||
);
|
||||
|
||||
const spotify = spotifyService();
|
||||
if (spotify) {
|
||||
registry.register({
|
||||
spec: {
|
||||
name: "media.spotify_play",
|
||||
description:
|
||||
"Search Spotify and start playing the best match on the user's Spotify (this phone or another of their Spotify devices). Use for requests like \"play Daft Punk\" (artist), \"play the album Discovery\" (album) or \"play Harder Better Faster Stronger\" (track).",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: { type: "string", description: "What to search for, e.g. an artist, album, song or playlist name." },
|
||||
kind: {
|
||||
type: "string",
|
||||
enum: ["track", "album", "artist", "playlist"],
|
||||
description: "What kind of thing to play (default track).",
|
||||
},
|
||||
},
|
||||
required: ["query"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
availability: "always",
|
||||
// Search, then play (with a device lookup when none is active).
|
||||
timeoutMs: 20_000,
|
||||
},
|
||||
handler: async (args) => {
|
||||
const query = String(args?.query ?? "").trim();
|
||||
if (!query) return err("media.spotify_play requires a query");
|
||||
const kind = ["track", "album", "artist", "playlist"].includes(args?.kind) ? args.kind : "track";
|
||||
try {
|
||||
return ok(`Playing ${await spotify.playSearch(query, kind)} on Spotify.`);
|
||||
} catch (error) {
|
||||
return err(String((error as Error)?.message ?? error));
|
||||
}
|
||||
},
|
||||
// Offered once signed in from the Music app menu.
|
||||
isAvailable: () => spotify.account() !== null,
|
||||
});
|
||||
}
|
||||
|
||||
// Android notification tools are still platform-specific.
|
||||
if (global.isIOS) return;
|
||||
|
||||
@@ -198,8 +244,8 @@ function formatEvent(event: CalendarEvent): string {
|
||||
return `- ${event.title || "(untitled)"} — ${when}${location}`;
|
||||
}
|
||||
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
|
||||
const title = notification.title || "(no title)";
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string; customText?: string[] }): string {
|
||||
const title = notification.title || notification.customText?.join(" ") || "(no title)";
|
||||
const body = notification.text ? `: ${notification.text}` : "";
|
||||
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
|
||||
}
|
||||
|
||||
@@ -76,9 +76,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -126,12 +126,10 @@ export type DashboardSnapshot = {
|
||||
activeTextEditorToggleValue: boolean;
|
||||
activeTextEditorToggleVisible: boolean;
|
||||
/**
|
||||
* The keyboard dialog is up on the glasses (opened by the phone's keyboard
|
||||
* button): the phone shows its typing panel, with a send button per
|
||||
* destination in the glasses menu's row order.
|
||||
* The keyboard dialog is open (opened by the phone's keyboard button): the
|
||||
* phone shows its typing panel, driven by keyboardInputSession.
|
||||
*/
|
||||
keyboardInputActive: boolean;
|
||||
keyboardInputTargets: ReadonlyArray<{ id: string; label: string }>;
|
||||
evenAppConflictMessage: string;
|
||||
evenAppConflictWarningVisible: boolean;
|
||||
firmwareWarningMessage: string;
|
||||
@@ -1148,7 +1146,6 @@ class DashboardController {
|
||||
activeTextEditorToggleValue: this.activeTextEditorToggle?.setting.get() ?? false,
|
||||
activeTextEditorToggleVisible: this.activeTextEditorToggle !== null,
|
||||
keyboardInputActive: this.keyboardInput !== null,
|
||||
keyboardInputTargets: this.keyboardInput?.targets ?? [],
|
||||
evenAppConflictMessage: this.evenAppConflictMessage,
|
||||
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
|
||||
firmwareWarningMessage: this.firmwareWarningMessage,
|
||||
@@ -1347,23 +1344,9 @@ class DashboardController {
|
||||
shell.startKeyboardInput();
|
||||
}
|
||||
|
||||
/** The phone's typing panel changed: mirror the text on the glasses. */
|
||||
setKeyboardInputText(text: string): void {
|
||||
shell.noteUserActivity();
|
||||
this.keyboardInput?.setText(text);
|
||||
}
|
||||
|
||||
/** Send to one destination, or (no id: the IME's send key) the one highlighted on the glasses. */
|
||||
sendKeyboardInput(targetId?: string): void {
|
||||
if (targetId) {
|
||||
this.keyboardInput?.sendTo(targetId);
|
||||
} else {
|
||||
this.keyboardInput?.send();
|
||||
}
|
||||
}
|
||||
|
||||
discardKeyboardInput(): void {
|
||||
this.keyboardInput?.discard();
|
||||
/** The open keyboard dialog, for the phone's typing panel (KeyboardInputViewModel). */
|
||||
get keyboardInputSession(): KeyboardInputSession | null {
|
||||
return this.keyboardInput;
|
||||
}
|
||||
|
||||
setActiveTextEditorToggleValue(value: boolean): void {
|
||||
@@ -2150,10 +2133,10 @@ class DashboardController {
|
||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||
}
|
||||
|
||||
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
|
||||
private multilingualProvider(): VoiceProvider | undefined {
|
||||
/** The provider and language a continuous-capture window needs (Translate's), if any. */
|
||||
private transcriptionChoice(): TranscriptionChoice | undefined {
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if (options.multilingualProvider) return options.multilingualProvider;
|
||||
if (options.transcription) return options.transcription;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -2178,20 +2161,21 @@ class DashboardController {
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs a multilingual provider sets it for every capture
|
||||
// while it is open, so push-to-talk still shares its stream.
|
||||
const multilingual = this.multilingualProvider();
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: multilingual ?? voiceProviderSetting.get(),
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
// Whisper detects the language itself when the speech may be in any.
|
||||
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
|
||||
@@ -26,7 +26,7 @@ export type FirmwareInfo = {
|
||||
};
|
||||
|
||||
/** The Faceclaw firmware revision this build of the app needs. */
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 37;
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 38;
|
||||
|
||||
const FACECLAW_PREFIX = "Faceclaw/";
|
||||
const LEGACY_PREFIX = "EVENCFW";
|
||||
@@ -36,7 +36,7 @@ const LEGACY_PREFIX = "EVENCFW";
|
||||
// unrecognized (its layout may differ from what our patch set targets).
|
||||
export const BASE_STOCK_VERSION = [2, 3, 0, 24];
|
||||
export const BASE_STOCK_VERSION_TEXT = BASE_STOCK_VERSION.join(".");
|
||||
export const VALIDATED_STOCK_VERSION = [2, 3, 2, 14];
|
||||
export const VALIDATED_STOCK_VERSION = [2, 4, 0, 24];
|
||||
export const VALIDATED_STOCK_VERSION_TEXT = VALIDATED_STOCK_VERSION.join(".");
|
||||
|
||||
export type FirmwareExtension =
|
||||
|
||||
@@ -19,12 +19,12 @@ export type FirmwarePatchSet = {
|
||||
export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"base": "g2_2.3.0.24.bin",
|
||||
"baseSha256": "187ccf2bcc5c17a212106e8a376745511e8289c4232b634a7ea94b9bf25a0979",
|
||||
"outputSha256": "90b932552e987fc554e6f748e50989503aaff0a93a3915702eacd96e3bd99e9b",
|
||||
"outputSha256": "4aac023f49ec6965186d9106e328a516f20a2915ab400d561f662f984023a432",
|
||||
"patches": [
|
||||
{
|
||||
"offset": 1045797,
|
||||
"old": "38b584b0",
|
||||
"new": "5cf377bb",
|
||||
"new": "5cf371bb",
|
||||
"desc": "ring receiver entry: timestamped unfiltered reports under framebuffer lease"
|
||||
},
|
||||
{
|
||||
@@ -36,43 +36,43 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 1437923,
|
||||
"old": "fef788ff",
|
||||
"new": "fdf292f8",
|
||||
"new": "fdf2a2f8",
|
||||
"desc": "cfw_ancs_event"
|
||||
},
|
||||
{
|
||||
"offset": 1437951,
|
||||
"old": "fef7e8fb",
|
||||
"new": "fcf2c0fd",
|
||||
"new": "fcf2d0fd",
|
||||
"desc": "cfw_ancs_open"
|
||||
},
|
||||
{
|
||||
"offset": 1437969,
|
||||
"old": "fef7e7fb",
|
||||
"new": "fcf2dbfd",
|
||||
"new": "fcf2ebfd",
|
||||
"desc": "cfw_ancs_close"
|
||||
},
|
||||
{
|
||||
"offset": 1388387,
|
||||
"old": "39694d00",
|
||||
"new": "05597d00",
|
||||
"new": "25597d00",
|
||||
"desc": "connection-bound ANCS EUS write callback"
|
||||
},
|
||||
{
|
||||
"offset": 971497,
|
||||
"old": "fff717f8",
|
||||
"new": "6ef3a7fc",
|
||||
"new": "6ef3a1fc",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 971845,
|
||||
"old": "fef769ff",
|
||||
"new": "6ef3f9fb",
|
||||
"new": "6ef3f3fb",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 979579,
|
||||
"old": "fdf74ef8",
|
||||
"new": "6cf3defc",
|
||||
"new": "6cf3d8fc",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
@@ -99,6 +99,12 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"new": "a325",
|
||||
"desc": "_connectParamReq_impl: force requested mode to fast (0xa3); disables idle slow requests"
|
||||
},
|
||||
{
|
||||
"offset": 1058287,
|
||||
"old": "fff754fb",
|
||||
"new": "59f342fb",
|
||||
"desc": "bl ble_classify_link (only a <= 11.25 ms / latency-0 link is 'already fast')"
|
||||
},
|
||||
{
|
||||
"offset": 1130227,
|
||||
"old": "5ff43432",
|
||||
@@ -132,31 +138,31 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 975103,
|
||||
"old": "07f085fb",
|
||||
"new": "6df3acfc",
|
||||
"desc": "bl headup_gate (head-up sensor event -> EvenHub sys event 12 under lease)"
|
||||
"new": "6df3a6fc",
|
||||
"desc": "bl headup_gate (idle input gate: tap/long/release -> field-102 events, stock result kept)"
|
||||
},
|
||||
{
|
||||
"offset": 830549,
|
||||
"old": "19f03efb",
|
||||
"new": "90f35bfe",
|
||||
"desc": "bl gesture_ring_press_mode (observe ring touch-down, preserve stock dispatch)"
|
||||
"new": "90f37bfe",
|
||||
"desc": "bl gesture_ring_press_mode (ring touch-down 14, head up 12 / down 15; stock dispatch preserved)"
|
||||
},
|
||||
{
|
||||
"offset": 830795,
|
||||
"old": "1ff071ff",
|
||||
"new": "90f314fe",
|
||||
"new": "90f334fe",
|
||||
"desc": "bl gesture_press (owned event 9 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
"offset": 830887,
|
||||
"old": "1ff043ff",
|
||||
"new": "90f30efe",
|
||||
"new": "90f32efe",
|
||||
"desc": "bl gesture_short_long (Faceclaw lease gates event 11 vs stock Menu path)"
|
||||
},
|
||||
{
|
||||
"offset": 831763,
|
||||
"old": "1ff08dfd",
|
||||
"new": "90f344fc",
|
||||
"new": "90f364fc",
|
||||
"desc": "bl gesture_release (owned event 10 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
@@ -210,43 +216,43 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 4537963,
|
||||
"old": "",
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f084fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f22c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f094fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f24c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
|
||||
"desc": "append injected blobs to main-app payload"
|
||||
},
|
||||
{
|
||||
"offset": 779123,
|
||||
"old": "805a3900",
|
||||
"new": "48703a00",
|
||||
"new": "68703a00",
|
||||
"desc": "main-app subheader payload size (ps)"
|
||||
},
|
||||
{
|
||||
"offset": 152,
|
||||
"old": "005b3900",
|
||||
"new": "c8703a00",
|
||||
"new": "e8703a00",
|
||||
"desc": "main-app TOC entry size (ps + 128)"
|
||||
},
|
||||
{
|
||||
"offset": 779243,
|
||||
"old": "805a3904",
|
||||
"new": "48703a04",
|
||||
"new": "68703a04",
|
||||
"desc": "main-app preamble length (low 24 bits)"
|
||||
},
|
||||
{
|
||||
"offset": 779247,
|
||||
"old": "effdbcea",
|
||||
"new": "6a8cfe55",
|
||||
"new": "0b0c48b0",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
|
||||
},
|
||||
{
|
||||
"offset": 156,
|
||||
"old": "c0c6322a",
|
||||
"new": "7f295d98",
|
||||
"new": "52937d21",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
|
||||
},
|
||||
{
|
||||
"offset": 779127,
|
||||
"old": "c0c6322a",
|
||||
"new": "7f295d98",
|
||||
"new": "52937d21",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -4,7 +4,7 @@ import { acceptInput, resetRingInputFilter } from "../ui/input-monitor";
|
||||
import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup, readActiveNotifications } from '../native/notification-icons.ios'
|
||||
import { shouldShowNotificationOnGlasses } from '../native/notification-sources'
|
||||
import { launcherEntries } from '../apps/launcher'
|
||||
import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
||||
import { discardZeroedEvenHubPackage, getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
||||
import { isInstalledPackagePresent } from '../apps/evenhub/updates'
|
||||
import { openEvenHubStoreForPackage } from '../apps/evenhub'
|
||||
import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager'
|
||||
@@ -14,6 +14,7 @@ import { registerNavigateTools } from '../assistant/navigate-tools'
|
||||
import { registerRoamTools } from '../assistant/roam-tools'
|
||||
import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
||||
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
||||
import { bindBandwidthBenchmarkSession } from '../native/bandwidth-benchmark.ios'
|
||||
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { voiceActivity } from '../ui/shell/voice-activity'
|
||||
@@ -31,6 +32,7 @@ import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmwar
|
||||
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
|
||||
import { ALL_APPS } from '../apps/all-apps'
|
||||
import type { AppContext, AppDefinition, AppLaunchParams } from '../apps/app-definition'
|
||||
import { WorkerAppHost } from '../ui/shell/worker-window'
|
||||
@@ -65,7 +67,7 @@ const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 2000
|
||||
|
||||
export type SessionPhase = 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error'
|
||||
export type SessionState = { phase: SessionPhase; status: string; battery: number | null; charging: boolean | null;
|
||||
ring: boolean; frames: number; capabilities: string; leftVersion: string; rightVersion: string;
|
||||
ring: boolean; ringBattery: number | null; ringCharging: boolean | null; frames: number; capabilities: string; leftVersion: string; rightVersion: string;
|
||||
/** Why the connected firmware can't run Faceclaw (firmware-compat.ts), or '' when it can or is unknown. */
|
||||
firmwareWarning: string }
|
||||
|
||||
@@ -84,7 +86,7 @@ export class IosPreviewController {
|
||||
private communicator: FaceclawCommunicatorBridge | null = null
|
||||
private readonly startedCommunicators = new WeakSet<FaceclawCommunicatorBridge>()
|
||||
state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false,
|
||||
frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' }
|
||||
ringBattery: null, ringCharging: null, frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' }
|
||||
private notifications: AncsClient | null = null
|
||||
private readonly sessionOffs: (() => void)[] = []
|
||||
private stopping: Promise<void> | null = null
|
||||
@@ -131,6 +133,8 @@ export class IosPreviewController {
|
||||
private clockMinute = -1
|
||||
private shellRenderInProgress = false
|
||||
private shellRenderQueued = false
|
||||
/** The shell woke; the next shell render unblanks the compositor after submitting. */
|
||||
private unblankAfterShellRender = false
|
||||
private nextFrameId = 1
|
||||
private nextWorkerFrame = 0
|
||||
private inputQueue: Promise<void> = Promise.resolve()
|
||||
@@ -149,11 +153,12 @@ export class IosPreviewController {
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Windows that want continuous capture, and what each needs of it (a
|
||||
// recording, a multilingual provider). They share one capture, which runs
|
||||
// recording, a provider and language). They share one capture, which runs
|
||||
// while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
||||
// transcriptionKey of the running continuous capture's provider; one that no
|
||||
// longer matches what its windows need is drained and restarted.
|
||||
// transcriptionKey of the running continuous capture (provider and
|
||||
// language); one that no longer matches what its windows need is drained
|
||||
// and restarted.
|
||||
private continuousTranscription = ''
|
||||
// Continuous capture should (re)start once audio is available: a window
|
||||
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||
@@ -198,11 +203,16 @@ export class IosPreviewController {
|
||||
send: () => { if (this.active && !this.glassesLocked) session.send() },
|
||||
sendTo: id => { if (this.active && !this.glassesLocked) session.sendTo(id) },
|
||||
discard: () => session.discard(),
|
||||
get visibility() { return session.visibility },
|
||||
setVisibility: visibility => { if (this.active && !this.glassesLocked) session.setVisibility(visibility) },
|
||||
get history() { return session.history },
|
||||
} : null),
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
|
||||
openNotificationsWindow: () =>
|
||||
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
|
||||
onSwitcherBarHeightChanged: () => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
@@ -210,7 +220,12 @@ export class IosPreviewController {
|
||||
},
|
||||
onScreenStateChanged: on => {
|
||||
if (on) this.glance.dismiss()
|
||||
void this.display?.setScreenBlanked(!on).catch(error => this.fail(error))
|
||||
// A wake unblanks only once the next shell render has staged an awake
|
||||
// scene: unblanking composites the retained one, painted while asleep
|
||||
// with no chrome and depth 0, and that first frame is the one the
|
||||
// brightness fade-in starts on.
|
||||
this.unblankAfterShellRender = on
|
||||
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
|
||||
this.requestShellRender()
|
||||
},
|
||||
})
|
||||
@@ -442,6 +457,7 @@ export class IosPreviewController {
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||
} else this.syncContinuousCapture()
|
||||
this.syncRawMicSession()
|
||||
void (async () => {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
@@ -519,10 +535,19 @@ export class IosPreviewController {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
if (!display) return
|
||||
const startedAt = Date.now()
|
||||
const bytes = shell.paintScene()
|
||||
// Taken with the paint, so a render that painted before the wake can't unblank.
|
||||
const unblank = this.unblankAfterShellRender
|
||||
this.unblankAfterShellRender = false
|
||||
const frameId = this.allocateFrameId()
|
||||
await display.submitShellScene(bytes, Date.now() - startedAt, frameId)
|
||||
try {
|
||||
const startedAt = Date.now()
|
||||
const bytes = shell.paintScene()
|
||||
await display.submitShellScene(bytes, Date.now() - startedAt, frameId)
|
||||
} finally {
|
||||
// Even if the render failed, so the screen doesn't stay dark; not if it
|
||||
// went back to sleep meanwhile (that sleep re-blanked).
|
||||
if (unblank && shell.isScreenOn()) await display.setScreenBlanked(false)
|
||||
}
|
||||
// Backpressure: the next shell render waits for this one to reach the
|
||||
// glasses (the preview target resolves immediately; nothing transmits).
|
||||
await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS)
|
||||
@@ -601,6 +626,7 @@ export class IosPreviewController {
|
||||
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
|
||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||
if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) { this.requestShellRender(); return }
|
||||
const window = shell.foregroundWindow()
|
||||
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId)
|
||||
&& isOnSwitcherEdge(x, y, window?.heightMode ?? 'min', window?.appId)) {
|
||||
@@ -624,9 +650,15 @@ export class IosPreviewController {
|
||||
const host = ALL_APPS.find(candidate => candidate.appId === 'evenhub')
|
||||
if (installed && host) {
|
||||
try {
|
||||
if (isInstalledPackagePresent(installed.packageId))
|
||||
await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed)
|
||||
else await openEvenHubStoreForPackage(this.buildAppContext(host), installed)
|
||||
if (isInstalledPackagePresent(installed.packageId)) {
|
||||
try { await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed); return }
|
||||
catch (error) {
|
||||
// Checked only after a failure: it reads the whole package again.
|
||||
if (!discardZeroedEvenHubPackage(installed.packageId)) throw error
|
||||
console.log(`[ios-preview] Discarded zeroed package for ${installed.packageId}; opening store page`)
|
||||
}
|
||||
}
|
||||
await openEvenHubStoreForPackage(this.buildAppContext(host), installed)
|
||||
}
|
||||
catch (error) { this.fail(error) }
|
||||
}
|
||||
@@ -656,6 +688,7 @@ export class IosPreviewController {
|
||||
appId: app.appId, apps: ALL_APPS, actions: this.actions,
|
||||
launchApp: (id, params) => this.launchApp(id, params), uninstallApp: id => this.uninstallApp(id),
|
||||
launchInProcessApp: (id, surface, create) => this.launchInProcessApp(id, surface, create),
|
||||
openInProcessAppInBackground: (id, surface, create) => this.openInProcessAppInBackground(id, surface, create),
|
||||
ensureWorkerHost: create => this.ensureWorkerHost(app.appId, create),
|
||||
submitWindowFrame: (id, planes) => this.submit(id, planes),
|
||||
setWindowSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
|
||||
@@ -667,14 +700,44 @@ export class IosPreviewController {
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): Promise<void> {
|
||||
const existing = this.inProcessApps.get(windowId)
|
||||
if (existing) { shell.focusWindow(windowId); existing.requestRender(); this.requestShellRender(); return }
|
||||
// Apps can request a render while their factory runs (Nightscout's tray
|
||||
// subscription does). The surface only exists once the factory returns.
|
||||
// The explicit render below supplies a fresh frame after configuration.
|
||||
let surfaceReady = false
|
||||
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
||||
await this.configureWindow(app.window)
|
||||
surface.ready = true
|
||||
shell.registerWindow(app.window); shell.focusWindow(windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
}
|
||||
/**
|
||||
* An in-process singleton app's window, opened in the background (neither
|
||||
* foregrounded nor focused) unless it is open already, for the shell to
|
||||
* bring forward itself: the Notifications window under a notification
|
||||
* selected in the switcher, or an app widget's window under the widget.
|
||||
* Display calls run in order and frames wait on displayReady before
|
||||
* queueing, so the surface configure queued here lands ahead of any frame
|
||||
* and of the shell foregrounding the window, which it may do at once.
|
||||
*/
|
||||
private openInProcessAppInBackground(windowId: string, surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): { window: InProcessWindow; opened: boolean } {
|
||||
const existing = this.inProcessApps.get(windowId)
|
||||
if (existing) return { window: existing, opened: false }
|
||||
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
||||
void this.configureWindow(app.window).catch(error => this.fail(error))
|
||||
surface.ready = true
|
||||
shell.registerWindow(app.window)
|
||||
return { window: app, opened: true }
|
||||
}
|
||||
/**
|
||||
* Create an in-process singleton app's window (not yet registered with the
|
||||
* shell). Its frames reach the compositor only while `surface.ready`: apps
|
||||
* can request a render while their factory runs (Nightscout's tray
|
||||
* subscription does), before the caller has configured the surface.
|
||||
*/
|
||||
private createInProcessApp(windowId: string, surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): { app: InProcessWindow; surface: { ready: boolean } } {
|
||||
const surface = { ready: false }
|
||||
const app = create({
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
|
||||
actions: this.actions, submitFrame: planes => surface.ready ? this.submit(surfaceId, planes, () => surface.ready) : Promise.resolve(),
|
||||
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
||||
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
|
||||
removeSurface: () => { surface.ready = false; this.removeWindowSurface(surfaceId) },
|
||||
reconfigureSurface: () => {
|
||||
const window = shell.getWindows().find(w => w.windowId === windowId)
|
||||
if (window) void this.configureWindow(window).catch(error => this.fail(error))
|
||||
@@ -683,10 +746,7 @@ export class IosPreviewController {
|
||||
onClosed: () => { this.inProcessApps.delete(windowId) },
|
||||
})
|
||||
this.inProcessApps.set(windowId, app)
|
||||
await this.configureWindow(app.window)
|
||||
surfaceReady = true
|
||||
shell.registerWindow(app.window); shell.focusWindow(windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
return { app, surface }
|
||||
}
|
||||
private ensureWorkerHost(appId: string, create: () => Worker): WorkerAppHost {
|
||||
const existing = this.appHosts.get(appId)
|
||||
@@ -738,14 +798,22 @@ export class IosPreviewController {
|
||||
this.pushBrightness()
|
||||
this.maybeLaunchOnboarding(this.state)
|
||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
|
||||
this.syncRawMicSession()
|
||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
this.syncContinuousSession()
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||
this.pushBatteryLevels()
|
||||
this.syncPresence()
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
}
|
||||
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
|
||||
private pushBatteryLevels(): void {
|
||||
const connected = this.state.phase === 'connected'
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
|
||||
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
|
||||
}
|
||||
async connect(): Promise<void> {
|
||||
if (!this.active || this.connecting || this.stopping) return
|
||||
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
|
||||
@@ -754,7 +822,8 @@ export class IosPreviewController {
|
||||
this.connecting = true
|
||||
this.sessionCreated = true
|
||||
try {
|
||||
this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false }
|
||||
this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false,
|
||||
ringBattery: null, ringCharging: null }
|
||||
this.firmwareCompatible = false
|
||||
this.update('connecting', 'Finding configured devices…')
|
||||
let identifiers
|
||||
@@ -796,16 +865,21 @@ export class IosPreviewController {
|
||||
this.notifications = notifications
|
||||
bindIosNotifications(notifications)
|
||||
bindCompassSession(communicator)
|
||||
bindBandwidthBenchmarkSession(communicator)
|
||||
communicator.setRequiresAncs(true)
|
||||
this.sessionOffs.push(
|
||||
communicator.onStateChange(state => this.handleSessionState(communicator, state)),
|
||||
communicator.onRingEvent(input => this.handleRingEvent(input)),
|
||||
communicator.onBatteryState(state => {
|
||||
if (this.communicator !== communicator) return
|
||||
// Unlike the headset level, an unavailable ring reading (-1: ring away or old
|
||||
// firmware) clears the indicator, as on Android.
|
||||
const ringBattery = state.ringBattery ?? -1, hasRingBattery = Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100
|
||||
this.state = { ...this.state, battery: state.battery >= 0 && state.battery <= 100 ? state.battery : this.state.battery,
|
||||
charging: state.chargingStatus >= 0 ? state.chargingStatus > 0 : this.state.charging,
|
||||
ring: (state.ringBattery ?? -1) >= 0 || this.state.ring }
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||
ring: hasRingBattery || this.state.ring,
|
||||
ringBattery: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null }
|
||||
this.pushBatteryLevels()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
}),
|
||||
@@ -934,6 +1008,9 @@ export class IosPreviewController {
|
||||
for (const off of this.sessionOffs.splice(0)) off()
|
||||
this.notifications = null
|
||||
this.communicator = null
|
||||
bindBandwidthBenchmarkSession(null)
|
||||
// Releases an EvenHub app's mic before the session closes.
|
||||
this.syncRawMicSession()
|
||||
this.glance.reset()
|
||||
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
||||
}
|
||||
@@ -977,7 +1054,7 @@ export class IosPreviewController {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.syncContinuousRecording()
|
||||
@@ -1047,6 +1124,15 @@ export class IosPreviewController {
|
||||
}
|
||||
if (current()) this.syncContinuousClaim()
|
||||
}
|
||||
/**
|
||||
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
|
||||
* unlocked glasses session, as speech capture does; the EvenHub mic router
|
||||
* decides which app hears it and when.
|
||||
*/
|
||||
private syncRawMicSession(): void {
|
||||
const ready = this.state.phase === 'connected' && !this.glassesLocked
|
||||
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
|
||||
}
|
||||
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||
private syncContinuousClaim(): void {
|
||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||
@@ -1058,15 +1144,17 @@ export class IosPreviewController {
|
||||
}
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs a multilingual provider (Translate) sets it for every
|
||||
* capture while it is open, with Whisper left to detect the language.
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
|
||||
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
@@ -25,6 +25,7 @@ import { EVENHUB_RUNTIME_FONT_FILENAME, isEvenHubFontAsset } from "../g2/firmwar
|
||||
import { lvglMetrics, lvglGlyph } from "../native/lvgl-font";
|
||||
import { GrayImage, grayToNibble, type FwTextFont, type PlacedFwText, type PlacedFwTextGlyph } from "./image";
|
||||
import { TtfFont } from "./ttf-font";
|
||||
import { decodeBase64 } from "../util/base64";
|
||||
|
||||
const CJK_ASSET_PATH = "fonts/source-han-sans/SourceHanSansSC-Light-20.lvgl.bin";
|
||||
|
||||
@@ -55,25 +56,6 @@ type CjkGlyph = {
|
||||
bitmap: Uint8Array;
|
||||
};
|
||||
|
||||
const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
function base64Decode(text: string): Uint8Array {
|
||||
const clean = text.replace(/[^A-Za-z0-9+/]/g, "");
|
||||
const out = new Uint8Array(Math.floor((clean.length * 3) / 4));
|
||||
let bits = 0;
|
||||
let bitCount = 0;
|
||||
let outIndex = 0;
|
||||
for (let i = 0; i < clean.length; i++) {
|
||||
bits = (bits << 6) | BASE64_ALPHABET.indexOf(clean[i]!);
|
||||
bitCount += 6;
|
||||
if (bitCount >= 8) {
|
||||
bitCount -= 8;
|
||||
out[outIndex++] = (bits >> bitCount) & 0xff;
|
||||
}
|
||||
}
|
||||
return out.subarray(0, outIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Codepoint ranges whose glyphs the font asset merged from the flash CJK
|
||||
* font with a baseline-shifted ofsY (see scripts/build-evenhub-font.mjs).
|
||||
@@ -144,7 +126,7 @@ export class EvenHubFont implements FwTextFont {
|
||||
this.useTtfFallback = useTtfFallback;
|
||||
this.lineHeight = asset.lineHeight;
|
||||
this.baseline = asset.baseline;
|
||||
this.bitmap = base64Decode(asset.bitmapBase64);
|
||||
this.bitmap = decodeBase64(asset.bitmapBase64);
|
||||
this.glyphs = new Map();
|
||||
for (const key of Object.keys(asset.glyphs)) {
|
||||
this.glyphs.set(Number(key), asset.glyphs[key]!);
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Generated by scripts/generate-pebble-icons.cjs; do not edit by hand.
|
||||
//
|
||||
// Pebble notification icons from https://github.com/pebble-dev/iconography
|
||||
// (commit 16a9ea42ea7ac079127af2ecc54f453dfbc314c8): the original Pebble Technology icons and
|
||||
// Rebble community additions, licensed under the Apache License 2.0 (see
|
||||
// app/graphics/pebble-icons.LICENSE). Modified: rasterized at 25x25 and
|
||||
// inverted for a black display (Pebble draws them black on white).
|
||||
//
|
||||
// Each icon is SIZE x SIZE pixels at 2 bits each (0, 1, 2 = coverage 0,
|
||||
// 128, 255), four pixels to a byte with the first in the high bits,
|
||||
// row-major, base64-encoded.
|
||||
|
||||
export const PEBBLE_ICON_SIZE = 25;
|
||||
|
||||
export const PEBBLE_ICON_DATA = {
|
||||
generic: "AAACgAAAAAAAoAAAAAAAKAAAAAACqqgAAAACqqqAAAACoACoAAACoAAKgAACoAAAqAAAoAAACgAAKAAAAoAACgAAAKAAAoAAACgAAKAAAAoAACgAAAKAACoAAACoACoAAAAKgCqqqAAAqAqqqgAACgKAAAAAAoCgAAAAAKAqqqqqqqgKqqqqqqoAAAAABkAAAAAAAKAAAAAAACgAAA==", // pebbleos/Pebble_25x25_Generic_notification.svg
|
||||
activity: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAAAACkAGkAAAAqkKqQAAAKaqhqAAACgagqpAAAoAQaakAAKAABGqkACgAAClqkAoAAAUZqQKqpAAKBqSlWqqqgBooAFVWkAKKAAAAZACilVVValVpqqqqqqqqQAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Activity.svg
|
||||
alarmClock: "AAAAAAAAAAAAAAAAAAoAAACgAAqCqqgqAAqCqqqCoAqCoACoKgKCoAAKgoACoAoAqAACoAKACoACoACgAKgAoAAoAAoAKAAKAAKACgAKgACgAoAKgAAoAKAKgAAKACgKgAACgAqCgAACoACoAAACoAAKgAACoAACqAACqAABmoACpkAAZKqqoZAAKAqqoCgACgAAAAoAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Alarm_clock.svg
|
||||
appStore: "AAAAAAAAAAACqoAAAAACqqgAAAACooqAAAACoKCoAAAAoCgKAAAAKAoCgAAACgKgoAAAAoAqKAAAAKgCqgAAAAqAKgAAAACqqgAAAAAKqgAABAAAAAAARpQAAAAFpGqqqqqqpAGqqqqqkAAoAAAAoAAKAAAAKAACgAAACgAAoAAAAoAAKAAAAKAACqqqqqgAAqqqqqoAAAAAAAAAAA==", // icons/25px_Appstore.svg
|
||||
calendar: "AAKAAKAAAACgACgAAKqqqqqqgCqqqqqqoAoCgACgKAKAoAAoCgCiqoCqooAoqqAqqKAKAAAAACgCgAAAAAoAoAAAAAKAKAAACgCgCgAABoAoAoAABoAKAKAKBoACgCgCkpAAoAoAKpAAKAKABpAACgCgAKAAAoAoAAAAAKAKAAAAACgCgAAAAAoAqqqqqqqAKqqqqqqgAAAAAAAAAA==", // pebbleos/Pebble_25x25_Calendar.svg
|
||||
car: "AAAAAAAAAAqqqqpAAAKqqqqgAAGQAAAoAACgAAAGQAAoAAAAoAAZAAAAKQAKAAAABkCqgAAAAKqqqqqqqqqgqqqqqqqAKAKACgCgCgCgAoAoAqqoAKqqAKqqACqqgCgAKqAAoAoACqgAKAKAAAAACgCqqqqqqoAqqqqqqqAAoKACgoAAKCgAoKAACgoAKCgCqqqqqqqqqqqqqqqqgA==", // pebbleos/Pebble_25x25_Car_rental.svg
|
||||
contacts: "AAAAAAAAACqqqqqqAAqqqqqqgAKAAAAAoAqqAAAAKAKqgAAACgAKAAAAAoACgAAAAKAAoAApACgAKAAaQAoACgAGkAKAKqgAAACgCqoAGAAoACgAGoAKAAoAGqgCgAKACqpAoACgAAAAKAAoAAAACgCqoAAAAoAqqAAAAKAAoAAAACgAKAAAAAoACqqqqqqAAqqqqqqgAAAAAAAAAA==", // icons/25px_Contacts_book.svg
|
||||
dinner: "AAAAAAAAAAAqqqAAAAAqqqoAAAKqAAKqAAKqAAAqoAKigAAKKgKgoAACgqCgKgACoCgoAqACoAoKACgAoAKCoAAAAAKgKgAAAAKgAqoAACqgACqAAAqgAACgAAKAAAAooCigAAAKKAooAAACigKKAAAAqqqqgAAAKqqqoAAACgAAKAAAAoAACgAAAKqqqoAAACqqqqAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Dinner_reservation.svg
|
||||
email: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqqoqqqqqqqqKqAAAACqiioAAACoooKgAACoKKAqAACoCigCoACoAooAKgCoAKKAAqCoACigACqoAAooAoKoKAKKAqAoCoCigqAAAKgooqAAAAqKKqAAAACqiqAAAAAKoqqqqqqqqKqqqqqqqgAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Generic_e-mail.svg
|
||||
flight: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAABqoAAAAAWpagqoABqkCqKqQGqQCqpkpalAAoKZGqQAAACmQpAAAAAooAAAqoAKKAAAKqAChkAUAGQCoZAKACgCoGQCgAoCoAqqoAaqoAKqqAKqoAAACgCgAAAAAoBkAAAAAKqoAAAAACqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Scheduled_flight.svg
|
||||
glucose: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAACgCgAAAAooAoAAAAqKAKAAAAqCgCgAoAqAoAoAqgqAKAKAqqqACgCgqCqAAoAoqAKAAKAKKAAAACgCgAAAAAoAoAAAAAKAKKqoAACgCiqqAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Glucose_monitor.svg
|
||||
heart: "AAAAAAAAAAKAACgAAAKoACqAAAKqgCqoAAKgqCoKgAKgCqoAqAKgAKoACoCgAAoAAKAoAAAAACgKAAAAAAoCgAAAAAKAoAAAAACgKAAAAAAoCoAAAAAqAKgAAAAqAAqAAAAqAACoAAAqAAAKgAAqAAAAqAAqAAAACoAqAAAAAKgqAAAAAAqqAAAAAACqAAAAAAAKAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Heart.svg
|
||||
incomingCall: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqgAACgAAAoAAAoqqqKAAAKKqqigAACigAooAAAooAKKCgAKKACigqACigAooCoAoqqqKAKgKKqqigAqCgAAAoACgoqiqKAAAKKoqiqqgCgAAAqqoAoqiqKAAAKKoqigAKCgAAAoAKgoAAAKAKgKAAACgKgCgAAAoKgAqqqqqCgAKqqqqgA==", // pebbleos/Pebble_25x25_Incoming_call.svg
|
||||
location: "AAACqqqoAAAAqqqqAAAAKAAqAAAACgAqAAAAAoAqAAAAAKAKgAAAACgAqAAAAAoACoAAAAKqqqgAAACqqqoAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAACqqqqAAACqqqqoAACoAoAKgACoAKAAqACoACgACoCoAAAAAKiqqqqqqqqqqqqqqqqgA==", // pebbleos/Pebble_25x25_Location.svg
|
||||
microphone: "AAAqqoAAAAAqqqgAAAAqCgqAAAAKBpSgAAACqqqoAAAAqmmqAAAAKAoCgAAACqqqoAAAAqqqqAAAAKAoCgAAACgKAoAAAAqmmqAAAAKqqqgAAACgaQoAAAAqCgqAAAACqqqAAACgKqqAoAAqAAAAqAACoAAAqAAAKqqqqAAAAqqqqAAAAAAoAAAAAAAKAAAAAAAqqAAAAAAKqgAAAA==", // icons/25px_Microphone.svg
|
||||
missedCall: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqiqoCgAAAoqqAoqqqKKoAKKqqiiqgCigAoooqAooAKKKCoKKACigAKiigAooACooqqqKACoKKqqigCoCgAAAoCoAoqiqKCoAKKoqiioACgAAAqoAAoqiqKoAAKKoqigAACgAAAoAAAoqiqKAAAKKoqigAACgAAAoAAAqqqqqAAAKqqqqgA==", // pebbleos/Pebble_25x25_Missed_call.svg
|
||||
movie: "AAAAAAAAAAAAAAAAAKqqqqqqACqqqqqqgAoqACoAoAKCoAKgKACgKgAqCgAoAqACooAKqqqqqqACqqqqqqgAoAqACooAKAqACoKACgqACoCgAoqACoAoAKqqqqqqACqqqqqqgAoAAAAAoAKAAAAAKACgAAAACgAoAAAAAoAKqqqqqqACqqqqqqgAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Movie_event.svg
|
||||
music: "AAKQAAAAAACqkAAAAAApqpAAAAAKAaqQAAACgAGqkAAApAABqoAAKqQAAaAACmqkACgAAoBqpAoKqqAAaqaCqqgAAGqgoAoAAABoKAKAAAAKCgCgAAACgoAoAAAAoKAKAAKqqCqqgACqqgqqoAAoAoAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACqqgAAAAAqqoAAAAAAAAAA==", // icons/25px_Music_note.svg
|
||||
news: "AAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqoAqqqqqqqAKAAAAACgCgAAAAAoAoKqqqoKAKCqqqqCgCgAAAAAoAoKqqqgKAKCqqqoCgCgAAAAAoAoKqqqAKAKCqqqgCgCgAAAAAoAoKqqAAKAKCqqgACgCgAAAAAoAoAAAAAKAKqqqqqqgCqqqqqqoAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_News_event.svg
|
||||
payBill: "AAAAAAAAAAAAAAAAAAAAAAAAApQAAAAAWqqlAABaqplqqAKqpZZAWgClAGSgAZBkACgoACgoAAoKqqqpAAZCqqqqQAGQoAACgAGgKAACkAaoCgACpBqqAoACqmqSgKACoqpAoCgAqCoAKAoACqoACgKAAKoAAoCgAAoAAKAoAAAAACgKqqqqqqoCqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Pay_bill.svg
|
||||
phone: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqgAACgAAAoAAAoqqqKAAAKKqqigAACigAooAAAooAKKCgAKKACigqACigAooCoAoqqqKAKgKKqqigAqCgAAAoACgoqiqKAAAKKoqiqqgCgAAAqqoAoqiqKAAAKKoqigAKCgAAAoAKgoAAAKAKgKAAACgKgCgAAAoKgAqqqqqCgAKqqqqgA==", // pebbleos/Pebble_25x25_During_phone_call.svg
|
||||
podcast: "ACgAAAAAAAoAAAAAAAZAAAAAAAGQAAAAAACgAAAAAAAoAAAAAAAZAAAAAAAGQAAAAAAqqqqqqqoKqqqqqqqCgAAAAACgoAAAAAAoKAqoAqqKCgqqgKqigoqAqAAAoKKACgAAKCigAoAACgooAKAAAoKKgKgAAKCgqqgAACgoCqgAAAoKAAAAAAKCgAAAAACgqqqqqqqoKqqqqqqqAA==", // pebbleos/Pebble_25x25_Radio_show.svg
|
||||
reminder: "AAAACqoAAAAAAqqCgAAAAKCioAAAACgqoAAAAAoKoAAAAAKqoAAAAACqqAAqqqqoKoAKqqqqCqgCgoKCgoqAoKCgoKCgKCgoKCgACgoKqqqAAoKCqqqgAKqqoAAoACqqqAAKAAqACqoGgACoAqqGgAAKgAACkAAAoAACkAAAKAAAoAAACgAAKAAAAoAACgAAAKqqqoAAACqqqqAAAA==", // pebbleos/Pebble_25x25_Generic_reminder.svg
|
||||
settings: "AAAKqqAAAAACqqgAAAAAoAoAAqqqqAKqqqqqqgCqqoAAAoAoAAAAAKqqAAAAACqqgAAAAAAAAAAAqqoAAAAAKqqAAAAACgCgAAAAqoAqqqqqqqAKqqqqoCgCgAAAAAqqoAAAAAKqqAAAAAAAAAAKqqAAAAACqqgAAAAAoAqqqqqqqAKqqqqqqgCgAAAAAoAoAAAAAKqqAAAAACqqgA==", // pebbleos/Pebble_25x25_Settings.svg
|
||||
shopping: "AAAAaAAAAAABqoAAAAACqaAAAAAApCgAAAAAKAoAAAAACgKAAACqqqqqqoAqqqqqqqAGgAAAAGQApAAAACgAGgAAAAoAApAAAAZAAGgAAAKAAAoAAACgAAZAAABkAAKAAABoAACgAAGqQABkAAKpoAAoAACkKQAKAAAABoAGQAAAAKQCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // icons/25px_Shopping_bag.svg
|
||||
sms: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqqAqqqqqqqoCoAAAAAKgKgAAAAAoAoAAAAAKAKCqqqqCgCgqqqqgoAoAAAAAKAKAAAAACgCgqqgAAoAoKqoAAKAKAAAAACgCgAAAAAoAqAAAAAqACqqqqqqAAKqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // icons/25px_SMS.svg
|
||||
stocks: "AAAAAAAACgAAAAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAoAAKAAAAqigCgAAAqqqAoAAAqCqoKAoAqCoKigqgqAqCooqqqACqoKqCqAAKoCqAKAAAoAqAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Stocks_event.svg
|
||||
tv: "AAAAAAoAAAAAAAqAAAAKAAqAAAACoAqAAAAAKgqAAAAAAqqAAAKqqqqqqqCqqqqqqqgoAAAAAAoKKqqqqqKCiqqqqqigooAAAAooKKAAAAKKCigAAACigooAAAAooKKAAAAKKCiqqqqqigoqqqqqooKAAAAAAKCgACqgACgoAAqoAAoKAAAAAAKCqqqqqqqgqqqqqqqoAAAAAAAAAA==", // pebbleos/Pebble_25x25_TV_show.svg
|
||||
voicemail: "AABAUAAAAABkaAAAAAAZKQAAAAAKKQAAAAKCigEAAACoABaQAABqgCqQAAApqApAAAAKCoAAAAAGQKgAAAACgAqAAAAAoBqgAAAAZCqQAAAAKQqAAAAACoBpAKAAAKgGkGoAAAaQaSqgAABpBqoqAAAGkCpCoAAAaQKAKgAABpAABqAAAGkAGqgAAAKhqpAAAAAqqQAAAAACkAAAAA==", // icons/25px_Phone.svg
|
||||
warning: "AAAAAAAAAAAAoAAAAAAAaQAAAAAAKoAAAAAAGqQAAAAACgoAAAAABkGQAAAAAoooAAAAAZKGQAAAAaCgpAAAAKQoGgAAAGgKApAAACkCgGgAABoAoApAAApAKAGgAAZAAAAZAAKAAoACgAGQAKAAZACgACgACgCkAAoAAaBoAAAAACkqqqqqqqqKqqqqqqqgAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Generic_warning.svg
|
||||
weather: "AAAAAAKAAAAAAACgAAAACgAoAAAAAqAKAAAAACoAAAAAAAKKqgAAAAAKqqAAAAAKgCoAAKqCgAKAAKqooACgAKgKqAAoKqgAqqqqKqoACqqqqgKgAAACqgAoAAAAKoAAAAAACqAAAAAAAqgAAAAAAKqAAAAAAKiqqqqqqqgKqqqqqqgAAAAAAAAAKCgoAAAACgoKAAAAAoKCgAAAAA==", // pebbleos/Pebble_25x25_Generic_weather.svg
|
||||
airmail: "AAAAAAAAAAAAAAAAAAAqqqoAAAAqqqqgAAAqAAAqAAAqAAACoAAqAAAAKgAqAAAAAqAKkAAAAagCqkAABqoApqkAGqaAKBqkapCgCgBqqkAoAoABqQAKAKAAAAACgCgAAAAAoAoAAAAAKAKgAAAAKgAaQAAAKgABpAAAKgAAGkAAKgAAAKqqqgAAAAqqqgAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Airmail_25px.svg
|
||||
amazon: "AACqqgAAAACqqqAAAACoACoAAACoAAKgAAAqqoAoAAAKqqgKAAAAAAqCgAAAAACgoAAAAqqoKAAAAqqqCgAAAqAAAoAAAqAAAKAAAKAqqCgAACgqqgoAAAoKAoKAAAKCgKCoAACgqqgKgAAoCqgAqAAKgAAoKgKAqAAqqgqoCqqqqgqKgKqqCgqAqAAAAAqACqqqqqqAAKqqqqqAAA==", // pebbleos/Pebble_25x25_Amazon.svg
|
||||
appleShortcuts: "AAAKqAAAAAAKqoAAAAAKgKgAAAAKgAqAAAAKgACoAAAKgAAKgAAKgAAAqAAKgCqgCoACgCqqAKAAoCoCoCgAKgoAKioAAqAAAqoAACoAACoAACqgACqgACoqACoqAAoCoCoCgAKAKqoAoACoAqoAqAAKgAAAqAAAqAAAqAAACoAAqAAAAKgAqAAAAAqAqAAAAACqqAAAAAAKqAAAAA==", // brand-logos/Apple_Shortcuts_25px.svg
|
||||
bbcNews: "qqqqqqqqoAAAAAAAKCqgqoCqCgqqKqiqooKCigooKKCqoqqKACgqqKqigAoKCigooKKCqoqqKqigqoKqAqgoAAAAAAAKAAAKqqACgAACqqgAoAAAoAoAKACqqAKACgAqqgCgAoAKAoAoAKACgKqqACgAoCqqgAoAKqoKAAKACqqCgACgAACqoAAoAAAqqAAKAAAAAAACqqqqqqqqgA==", // brand-logos/BBC_News_25px.svg
|
||||
beeper: "AAqqqqAAAAqqqqoAAAqAAAKgAAqAAAAqAAqAAAACoAKAAAAAKACgAAAACgAoAAAAAoAKAAAAAKACoAAAAKgAKgAAAKgAAqAAAKgAAKqqoKgAACqqqKoAAAoAGaqgAAKACqgqAACgAqgCoAAoAagAKAAKAKgACgACgCgAAoAAoAAAAqAAKAAAAqAACgAAAqAAAqqqqqAAAKqqqqAAAA==", // brand-logos/Beeper_25px.svg
|
||||
bluesky: "AAAAAAAAAGgAAAApAqqAAAAqqKWoAAAqWigKgAAqAooAqAAqAKKACoAqACigAKgqAAooAAqqAAKKAACqAACigAAKAAAopAAAAAAaGgAAAAAKQpAAAAAGgGqoAAKqkAqqAACqoAKpAAAGqACkAAAAGgAoAAAAAoAKAAKAAKACgAGkACgAqACqACoACoBqkCoAAKqpaqoAAAqqCqoAAA==", // brand-logos/Bluesky_25px.svg
|
||||
discord: "AAAAAAAAAAAAAAAAAAAAAAAAAABoAACkAAKqqqqqoAClqqqpaABkAAAABkAoAAAAAKAKAAAAACgGQAAAAAZCgAAAAACgoCqgKqAoZAqoCqgGaAKKAooAqgCqgKqAKoAqoCqgCqAAAAAAAqkAAAAAAaqkCqqoBqhqmqqqmqQBqoAAqpAABoAACkAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Discord_25px.svg
|
||||
duolingo: "AAAAAAAAAAAAAAAAAAoAAAAoAKagAAAqaCqqAAAqqhqmoAAqappoKgAqCmqAAqAqAAUKqCgKCqgKqoAACqqKgKgACoCqgAoAAoAKoKKAAKKCqCigACigqgooAAooKoKKAAKKCqCiiqiigqoCqqqqAqKqqoCqqqAqoqqqKqAAAKqqgAAAAAoCgAAAAAKqoAAAAAAqoAAAAAAAAAAAAA==", // brand-logos/Duolingo_25px.svg
|
||||
ebay: "AAAqqgAAAAAqqqAAAAAqACoAAAAqAAKgAAAKAAAoAAACgAAKAAKqqqqqqqCqqqqqqqgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKgCgAoAqAqAoAKAqACqqqqqqAAKqqqqqAAA==", // brand-logos/eBay_25px.svg
|
||||
element: "AACqqoAAAAAqqqgAAAAKAAqAAAACgACoAAAAqqgKAACqqqqCgACqqqCgoACoACgoKACoAAoKCqqoCqqCgqqqCqqgqqgqgoAAKqoKoKAAAAKCqCqqAACgqgqqgqqoKqqgoKqoCqqoKCgACoAKCgoACoACgoKqqoAAoKqqqoAAKAqqgAAACoAAoAAAAKgAKAAAAAqqqgAAAACqqoAAAA==", // brand-logos/Element_25px.svg
|
||||
facebook: "AAAAAAAAAAAAAAAAAAAAAAAAAAqqqqqqgAKqqqqqoACgAAAAKAAoAAAACgAKAAAKgoACgAAKoKAAoAAKgCgAKAACgAoACgAAoAKAAoACqqCgAKAAqqgoACgAAoAKAAoAAKACgAKAACgAoACgAAoAKAAoAAKACgAKAACgAoACqqqqqqAAqqqqqqgAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Facebook_notification.svg
|
||||
facetime: "AAAoAAAAAAAKAAAAAoACgAKAAKgAoAKgAAqAKAKgAACoCgKgAAAKgoKgAAAAoACgAAAAAAAAAAAAAAAAAAAAKqqqoACgKqqqqgCoKgAAAqCqKgCqgCqqigCqqAKoooCoCoCoKKAoAKAoCigKACgKAooCgAoCgKKAqAqAqCigCqqAKooqAKqAKqqCoAAAKgqgKqqqqgCoAqqqqgAKAA==", // pebbleos/Pebble_25x25_Facetime_incoming.svg
|
||||
gmail: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqqqqiqqqqqqqooAqAAqAKKACoAqACigAKgqAAooKAqqAoKKCoCqAqCigqgKAqgooKqAAqoKKCioAqKCigoKgqCgooKCqqgoKKCiqqqKCigqoKCqgooKoAAKoKKCoAAAqCiqqqqqqqoqqqqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Gmail_notification.svg
|
||||
googleChat: "AAAAAAAAAAAAAAAAAKqqqqoAACqqqqqgAAoAAAAqAAKAAAACoACgAAAAKgAoCqqqgqAKAqqqoCgCgKAAKAoAoCgACgKAKAoAAoCgCgKAAKAoAoCqqqgKAKAqqqoCgCgKgAAAoAoCgAAAKAKAoAAACgCqqAqqqoAqqgqqqqAAAoqAAAAAAKqAAAAAACqAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Google_Chat_25px.svg
|
||||
googleMaps: "AAAAAKqgAAAAAKqqAKqqqqgCoKqqqqgAKqgAACgAAqgKoAoAAKoKqAKAACqKgACgAAqigqgqAAqooKoCoAqKKgqAKgqCgqqAKoKgoCqAKmWoKAAAKgqqCgAAKgKqgoAAKgBpoKAAKqAKKCgAKioCigoAKgKgAoKAKgAqAKCgKgACoCgoKgAAKgoKqgAAAqqAqqqqqqqACqqqqqqAAA==", // pebbleos/Pebble_25x25_Google_Maps.svg
|
||||
googlePhotos: "AAAAoAAAAAAAKgAAAAAACqAAAAAAAqoAAAAAAKKgAAAAqqgqAAAAqqoCoAAAqAKAKAAAqACgCgAAqACqAoAAqACqoKAAqqqoKqqqKqqqCqqqgAKCqoAKgACgKoAKgAAoAoAKgAAKAKAKgAACoCqqgAAAKgqqgAAAAqKAAAAAACqgAAAAAAKoAAAAAAAqAAAAAAACgAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Google_Photos.svg
|
||||
googleTasks: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAqqqAqAAAqqqoqAAAqAAKqAAAqAAAqAAAqAAAqoAAKAAAqKAACgAAqCgAAooAqAoAAKKgqAKAACgqqACgAAoCqAAoAAKAKAAKAACoAAAKgAAKgAAKgAAAqAAKgAAACqqqgAAAAKqqgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Google_Tasks_25px.svg
|
||||
homeAssistant: "AAAAAAAAAAAAKAAAAAAAKoAAAAAAKqgAAAAAKgqAAAAAKgCoAAAAKgAKgAAAKgAAqAAAKgqqCoAAKgKqgKgAKgCgoAqAKgAoKACoKgAKqgAKigACqoAAooAACgqqKKAAAoKqiiiqoKCgoooqqCgoKKKKCgoKqiiigoKKqoooqqCqgAKKKqoqgACigAKqgAAoqqqqqqqqKqqqqqqqgA==", // brand-logos/Home_Assistant_25px.svg
|
||||
instagram: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAqqgCgAAAKqoAoAKqoAKAKAKqqgCgCgKgAqAoAqqgACqqAKqgAAKqgCgoAACgoAoKAAAoKAKCgAAKCgCgoAACgoAoKgACoKAKAqACoCgCgCqqoAoAoAKqoAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Instagram.svg
|
||||
iosPhotos: "AAAKqAAAAAAKqoAAAAqqgKqoAAqqgAqqgAqBoAKQqAKAKACgCgCgBkBkAoApAKAoAaAKpBkZBqgKqpaKWqqKgGqqqkCqgAGqqQAKoAAKqAACqAAaqpAAqoBqqqpAqKqpaKWqqAqkGRkGqAKQCgKAGgCgBkBkAoAoAoAKAKAKgaACkKgAqqgAqqgACqqAqqgAAACqqAAAAAAKqAAAAA==", // pebbleos/Pebble_25x25_iOS_Photos.svg
|
||||
kakaoTalk: "AAAAAAAAAAAAAAAAAAKqqqqAAAKqqqqoAAKgAAAKgAKgAAAAqAKgAAAACoCgCqqqAKAoAqqqgCgKAACgAAoCgAAoAAKAoAAKAACgKAACgAAoCgAAoAAKAoAAKAACgKAACgAAoCoAAAAAqAKgAAAAqAAqAAAAqAACgKqqqAAAoaqqqAAAKaQAAAAACqQAAAAAAqQAAAAAAKAAAAAAAA==", // pebbleos/Pebble_25x25_KakaoTalk.svg
|
||||
kik: "AoAAAAAAAqgAAAAAAqqAAAAAAKCgAAAAACgoAAAAAAoKAKqAAAKCgKqgAACgoKgoAAAoKKgKAAAKCqgKgCgCgqgKgGqQoKgKgKlqKCgKgCgCigAKgAoAooAKgAKAKKACoACpaigAKgAGqQoKAqAAKAKCoCoAAACgqgKgAAAoKqAqAAAKCioCgAACqoKgoAAAKoAqqAAAAoACqgAAAA==", // pebbleos/Pebble_25x25_Kik.svg
|
||||
line: "AKqqqqqAAKqqqqqoAKgAAAAKgKgAAAAAqKgAAAAACqgAAAAAAKoKAoqoqiqCgKKqKoqgoCiiigKoKAoooqiqCgKKKKoqgoCiiigKoKooooqiqCqKKKKoqoAAAAAAKKgAAAAAKgqAAAAAKgCoAAAAKgAKqqqAKgAAqqqgKgAAAAAZKgAAAAAGagAAAAABqgAAAAAAKgAAAAAACgAAAA==", // pebbleos/Pebble_25x25_Line.svg
|
||||
linkedIn: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAACgCgKAAAAoAoCgAAAKAKAAAAACgCgKAAAAoAoCgqqAKAKAoKqoCgCgKCgKAoAoCgoCgKAKAoKAoCgCgKCgKAoAoCgoCgKAKAoKAoCgCgAAAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_LinkedIn.svg
|
||||
messenger: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAABqAACgAACqAABqQAGqQAApqAKqgAAZCpamkAAaAGqRoAAKQAKAoAAGgAAApAACgKAAaAACkapAKQABpqWoGQAAqqAKmgAAapAAakAAKoAAAoAAKkAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Facebook_messenger_notification.svg
|
||||
omniFocus: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgAAAAAAKqAAAAAAKioAAAAAKgKAAAAAKgKgAAAAKgKgACoAKgKgACqgKgKgACoqKgKgAAoCqgKgAAKgKgKgAAAqAQKgAAACoAKgAAAAKgKgAAAAAqKgAAAAACqgAAAAAAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/OmniFocus_25px.svg
|
||||
outlook: "AAAAAAAAAAAAFAAAAAAAapAAAAABqqpAAAAGqQapAAAapAAapABqkAAAapCqQAAAAaopAAAAAAaKAAAAAACikAAAAABqqqqoAAGqqqqqAAaqqAACgBqkqiqooGqQKoqqKqpACqKCiqkAAqigopqkAKoqqKBqgCqKqigBoAqgAAoAKAKqqqqAKgKqqqqgKgKgCqqqqqqgAKqqqqqgAA==", // brand-logos/Outlook_25px.svg
|
||||
reddit: "AAAAAAAAAAAAAAAAAAAAAAqgAAAAAAqqAAAAAAKCgAAAAACgoAAAAAKqqAAAAACmqAAAAABkAAAAAAAoAAAAqkaqpGqAqqqqqqqoqBpAAGkKqAAAAAAAqgAKACgAKqECgAoBKiqQoAKBqgKgAAAAKgAoAqqgCgAKQKqoBoACqQqoGqAAGqQAapAAAGqqqkAAAAGqqQAAAAAAAAAAAA==", // brand-logos/Reddit_25px.svg
|
||||
signal: "AAqqCqgAAAqqgqqAAAKAAACoAAACqqoKAAoCqqqgAAqCoAAqAoqCoAACoKqCoAAAKgqioAAAAqKooAAAACigKAAAAAooCgAAAAKKooAAAACgKKAAAAAoCigAAAAKKooAAAACiqKAAAACoqigAAACoKBkAAACoKgoAAACoKgKBqqqoCgGqqqqoAACqkAAAKAApAqoKqgAAAKqCqgAAA==", // brand-logos/Signal_25px.svg
|
||||
simplex: "AAAAAAAAAAAoAoAAAAAqgqgAAAAqqqqAAAAqCqCoAAAKgKAqAAAoqAAqKAAqipQqKoAqqqpKqqgqCqGQqgqKgKAoCgKgqAAqgAKgCoAqqAKgAqAKqgCoAqAAqgAKgqAoCgKAqKgqhkKoKgqqqpKqqgCqKlCoqgAKKgAKigAAKgKAqAAACoKoKgAAAKqqqgAAAAqgqgAAAACgCgAAAA==", // brand-logos/SimpleX_25px.svg
|
||||
slack: "AAAAGgAAAAAAqoAAAAAAKWWgAAGgBkqoAAqoAaKWQAKWWqgBoABkqpAKqAAaKQGikACqgBqoZAApAKqWSgAGWikAooAAqoZAKGWgKWSgBkqoAAooAaKWQAKGWqgBoABkqpAKqAAaKQGikACqgBqoZAApAKqWWgAGWikAqoAAqoZAKQAAKWWgAAAAAAqoAAAAAAKQAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Slack.svg
|
||||
snapchat: "AACqqgAAAACqqqAAAACoACoAAACoAAKgAAAoKCgoAAAKCgoKAAKqgoKCqoCqoKCgqqAoACgoACgKQAAAABoCqQAAAGqAGqAAAKpAAGgAACkAAAoAAAoAAAqAAAKgAAqAAAAqAAqAAAACoAqAAAAAKgqAAAAAAqKAAAAAACikAAAAABoqlAAAAWqAapAABqkAAWqqqpQAAAGqqkAAAA==", // pebbleos/Pebble_25x25_Snapchat.svg
|
||||
steam: "AAAAAAAAAAAAAAAAAAAAACqqAAAAACqqoAAAACoAKgAAACoqoqAAAAoqqigAAAKKAooAAACigKKAAAAooCigAAAaKqooAAAKAqoqAAKqQAAqKQKqgCqqCqqgAGqqABqpAKkAAAAqqCgAAAAKGgoAAAACgAKAAAAAqAKgAAAACqqgAAAAAKqgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Steam_25px.svg
|
||||
substack: "AAAAAAAAAAAAAAAAACqqqqqqAAqqqqqqgAKAAAAAoACgAAAAKAAoAAAACgAKqqqqqoACqqqqqqAAoAAAACgAKAAAAAoACqqqqqqAAqqqqqqgAKAAAAAoACgAAAAKAAoAAAACgAKAAAAAoACgAAAAKAAoAGpACgAKAaqpAoAChqkapKAAqqQAaqgAKpAAAaoACkAAAAaAAAAAAAAAAA==", // brand-logos/Substack_25px.svg
|
||||
swarm: "AAAAAKQAAAAAAKqAACkAAKmgACqpACgoACpaqAoKACoAGkKGgApAAaCmkAGgACkqoAApAAaKkAAGqqqgAAAAqqpABqkAAAAAaqqkAAACqqBqgAABpCgBoAAAqQoAKAAAakKACgAAKpBkAoAAGmgKAKAACkoCoCgABoKQKioAApApAqoAAKpGgGoAAAaqqqoAAAAaqqQAAAAAAAAAAA==", // brand-logos/Swarm_25px.svg
|
||||
tapo: "AAABkAAAAAACqgAAAAACoqAAAAACoCoAAAACoCqgAAACoCoqAAACoAoCoAACoAKAKgACoACgAqACoAAoACoCoAAqAAKhoAAqoAApoAAqKgACqAAqAqAAqgAqACoAKoAKAAKACqACgACgAqgAoAAoAKoAKAAKACqACgACgAqgAoAAoAKoAKAAKACqACgACgAqqqoAAqqqqqqAAKqqgA==", // brand-logos/Tapo_25px.svg
|
||||
teams: "AAKoAAAAAAKqgAAAAAKgqAAAAACgCgAAAAAoAoAqgAAKgqAqqAAAqqAKCgAACqACgoACqqqgqqACqqqqCqACoAACqqoCqqqoKqqgqqqqCgAqKAACgoACiiqooKAAooqqKCgAKKAoCgoACigKAoKAAooCgKCgAKKAoCgoACigAAoKAAoqqqqCoAKKqqqgKgKgCqqqqqqgAKqqqqqgAA==", // brand-logos/Teams_25px.svg
|
||||
telegram: "AAAAAAAAAAAAAAAAAAAAAAAAaAAAAAABqgAAAAAWqkAAAABqkpAAAAGqQKAAAAapAGQAAGqQABkAAapAAAoABqkAGgKAWpQBaoGQqkAGqoCgKqRappAoAGqqUaAZAABpAKAGQAAGQKQCgAABkGqRoAAAKCmqpAAACikGqAAAAaoAGgAAAGoAAAAAAApAAAAAAAKAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Telegram_app.svg
|
||||
threads: "AAKqqqgAAAKqqqqAAAKgAACoAAKgAAAKgAKgKqqAqAKgKqqoCoKgKgAKgqCgCoAAoqAoAKqoKqAKAKqqCqACgKgAACoAoKgAAAKgKCgqqAAoCgoKqgoKAoKCgoKCgKCgqqCgoCgoKqgoKAoKgAAqCgKAqAAqAoCoCqqqAqAKgKqqAqAAqAAAAqAACoAAAqAAAKqqqqAAAAqqqqAAAA==", // brand-logos/Threads_25px.svg
|
||||
tiktok: "AAAAAAAAAAAAKqgAAAAACqpAAAAAAoGgAAAAAKAqAAAAACgCpQAAAAoAKoAAAAKAAKAACqigoCgACqooKkoACoKKCqqACoCigpqgCoGooKAAAoGqKCgAAKCkCgoAACgoAoKAAAoKAKCgAAKCkGgoAACgaqkKAAAqBqkKgAACoAAKgAAAKgAKgAAAAqqqgAAAACqqgAAAAAAAAAAAAA==", // brand-logos/TikTok_25px.svg
|
||||
twitch: "AAqqqqqqAAqqqqqqgAqgAAAAoAqoAAAAKAqKAAAACgKCgCgKAoCgoAoCgKAoKAKAoCgKCgCgKAoCgoAoCgKAoKAKAoCgKCgAAAAoCgoAAAAqAoKAAAAqAKCgAAAqACgqqqqqAAoKqqqqAAKAAqAqAACgAKAqAAAqqgqqAAAKqoqqAAAAAKqAAAAAACqAAAAAAAqAAAAAAAKAAAAAAA==", // brand-logos/Twitch_25px.svg
|
||||
twitter: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWgAKQAACqqFqqkAApSqqKqpAZAKaRpaqWQAGgKQGqgACkApABoAAoAGgAAAAKAAaAAAACgACkAAAAoAAaAAAAKAABoAAAGgABqAAAGgACpAAAGkAGoAAACkAKoAAAWkACqpQWqoAABaqqpQAAAAFpQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Twitter_notification.svg
|
||||
unifiProtect: "AAAAAAAAAAAaqkAAAAAqqqgAAAAqQBqAAAAaCqCkAAAKSqoaAAACgoKCgAAAoKCgoAAAKCqoKAAACgKoCgAAAoAAAoAAAKAAAKAAACgCgCgAAAoAoAoAAAKAKAKAAACgAACgAAAoAAAoAAAKAAAKAAACgAACgAAAoAAAoAAAKAAAKAAACoAAKgAAAKqqqgAAAAqqqgAAAAAAAAAAAA==", // brand-logos/Unifi_Protect_25px.svg
|
||||
viber: "AAAAAAAAAAACqqgAAAAAqqqAAAAAAACoAAKAAAAKgAKoAKqAqAKqgCqoCoKgqAAKgKigCoAAqAooAKgoCoKKACoKgKCigCoAqCgooCoACgoKKAqAAAKCigCoAAAAoqAKgCgAACoAqCqAAAKgCqqoAAAqAKoKgAACoAoAqAAAKgAACoAAAqAAAqAAACoAAqAAAAKqqqAAAAAqqqAAAA==", // pebbleos/Pebble_25x25_Viber.svg
|
||||
waze: "AAAAAAAAAAACqqoAAAACqqqgAAACoAAqAAACoAACoAACoAAAKgACoAFBQqAAoACgoCgAKAAoKAoACgAKCgKAAoABQUCgAKAAAAAoqqgCgCgKKqoAqqoCgqAACqoAoCoAAAAAqAKgAAAAqAAqqgAAqAACqqAAqAAAoCqqqAAAKAqqqgAACgKKAoAAAqqiqqAAACqgKqAAAAAAAAAAAA==", // brand-logos/Waze_25px.svg
|
||||
weChat: "AAAAAAAAACqqqAAAACqqqoAAACoAAKgAACoAAAqAACoAAACoAAoKoKoKAAKCqCqCgACgAAAAoAAoAAAKqqgKAAAKqqqCgAAKgACoqAAKgAAKioACgAAAoKAAoqiqKCqqqKoqigqqqgAAAoKgAoAAAKCgAKgAAKgAAAqAACgAAACqqqoAAAAKqqqAAAAAAAKgAAAAAAAoAAAAAAAAAA==", // pebbleos/Pebble_25x25_WeChat.svg
|
||||
whatsApp: "AAAAAAAAAAAAAAAAAAAKqqoAAAAKqqqgAAAKgAAqAAAKgAACoAAKigAAKgAKiqAAAqAKiqoAACoCgoKgAAKAoKCoAACgKCioAAAoCgoqAAAKAoKCoKACgKCoKqoAoCgKgqqgKAoAqCgqCgKgCoAKioBoAKqqioAZAAqqioAKAAAACoAGlqAACoACqqqqqoAAqVKqqoAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_WhatsApp_notification.svg
|
||||
whatsAppBusiness: "AAAAAKqgAAAAACqoAAAqqqoKAAAqqqqCqgAqACqgqoAqAAoAAKAqKAKAACgqKoCqgqoqKqgqoKqKCgqAKCoCgoKgCqqAoKKgAqqgKCioAAAoCgoKgoAKAoKgqqgCgKAqCqqAoCgCoKCoKAqAKgAqKgGgAqqqKgBkACqqKgAoAAAAKgAaWoAAKgAKqqqqqgACpUqqqgAAAAAAAAAAAA==", // brand-logos/WhatsApp_Business_25px.svg
|
||||
yahooMail: "CqqqqqqgAqqqqqqoAKAAAAAKACgqoAACgAoKqKqAoAKAKiqgKACgAqqACgAoACqAAoAKAAKAAKACgAqqACgAoAKqgAoCqAAAAAKoqgACgACqKoACqAAqiqgCqoAqooqCoKgqKKCqoAqqCigKoACqAooCoAAKgKKCoAAAqCiioAAACooqoAAAAKqKqqqqqqqiqqqqqqqoAAAAAAAAAA==", // pebbleos/Pebble_25x25_Yahoo_mail.svg
|
||||
youtube: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqqoAKqqqqqqgKgAAAAAqCgAAAAACgoACkAAAoKAAqkAAKCgAKqkACgoAChqgAoKAAoBoAKCgAKGqACgoACqpAAoKAAqkAAKCgAKQAACgoAAAAAAoKgAAAAAqAqqqqqqqACqqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Youtube_25px.svg
|
||||
zoom: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqoABqqqqqqgGqoAAAAqaqqAAAACqkqgAAAAKQKoAAAAAACqAAAAAAAqgAAAAAAKoAAAAAACqAAAAAAAqgAAAAAAKoAAAAAACqgAAAApAoqAAAAKpKCoAAACqqgKqqqqoaoAqqqqqAaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Zoom_25px.svg
|
||||
} as const;
|
||||
|
||||
export type PebbleIconName = keyof typeof PEBBLE_ICON_DATA;
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,28 @@
|
||||
import { GrayImage } from "./image";
|
||||
import { decodeBase64 } from "../util/base64";
|
||||
import { PEBBLE_ICON_DATA, PEBBLE_ICON_SIZE, type PebbleIconName } from "./pebble-icons-data";
|
||||
|
||||
export { PEBBLE_ICON_SIZE, type PebbleIconName };
|
||||
|
||||
// 2-bit codes in pebble-icons-data.ts -> coverage.
|
||||
const LEVELS = [0, 128, 255, 255];
|
||||
|
||||
const cache = new Map<PebbleIconName, GrayImage>();
|
||||
|
||||
/**
|
||||
* One of the bundled Pebble notification icons (see pebble-icons-data.ts and
|
||||
* scripts/generate-pebble-icons.cjs), as a PEBBLE_ICON_SIZE square coverage
|
||||
* bitmap. Decoded once and shared: clone it before drawing on it.
|
||||
*/
|
||||
export function pebbleIcon(name: PebbleIconName): GrayImage {
|
||||
let icon = cache.get(name);
|
||||
if (!icon) {
|
||||
const bytes = decodeBase64(PEBBLE_ICON_DATA[name]);
|
||||
icon = new GrayImage(PEBBLE_ICON_SIZE, PEBBLE_ICON_SIZE, 0);
|
||||
for (let i = 0; i < icon.pixels.length; i++) {
|
||||
icon.pixels[i] = LEVELS[(bytes[i >> 2]! >> (6 - 2 * (i & 3))) & 3]!;
|
||||
}
|
||||
cache.set(name, icon);
|
||||
}
|
||||
return icon;
|
||||
}
|
||||
@@ -11,6 +11,11 @@ export interface WrapFont {
|
||||
|
||||
export type WrapTextOptions = {
|
||||
preserveLeadingWhitespace?: boolean;
|
||||
/**
|
||||
* A word too wide for a line of its own starts on the current line and is
|
||||
* split at the width, rather than first being pushed to a line of its own.
|
||||
* Words that fit on a line still wrap whole.
|
||||
*/
|
||||
breakLongWords?: boolean;
|
||||
};
|
||||
|
||||
@@ -89,17 +94,20 @@ function wrapParagraph(font: WrapFont, paragraph: string, targetWidth: number, o
|
||||
|
||||
while (start < paragraph.length) {
|
||||
let best = -1;
|
||||
let overflow = -1;
|
||||
for (const candidate of wrapOffsets) {
|
||||
if (candidate <= start) continue;
|
||||
if (measureSpan(widths, candidate) - measureSpan(widths, start) <= targetWidth) {
|
||||
best = candidate;
|
||||
continue;
|
||||
}
|
||||
overflow = candidate;
|
||||
break;
|
||||
}
|
||||
|
||||
const splitOffset = furthestFittingOffset(widths, start, targetWidth);
|
||||
if (opts.breakLongWords && splitOffset > start && (best < 0 || splitOffset > trimEndOffset(paragraph, best))) {
|
||||
if (opts.breakLongWords && splitOffset > start && splitOffset > best
|
||||
&& (best < 0 || (overflow >= 0 && wordWidth(paragraph, widths, best, overflow) > targetWidth))) {
|
||||
best = splitOffset;
|
||||
}
|
||||
if (best < 0) {
|
||||
@@ -157,6 +165,11 @@ function measureSpan(widths: Map<number, number>, offset: number): number {
|
||||
return widths.get(offset) ?? 0;
|
||||
}
|
||||
|
||||
/** Width of the word between two wrap offsets, without surrounding whitespace. */
|
||||
function wordWidth(text: string, widths: Map<number, number>, from: number, to: number): number {
|
||||
return measureSpan(widths, trimEndOffset(text, to)) - measureSpan(widths, skipLeadingWhitespace(text, from));
|
||||
}
|
||||
|
||||
function skipLeadingWhitespace(text: string, offset: number): number {
|
||||
let cursor = offset;
|
||||
while (cursor < text.length) {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Platform-neutral ambient-light types and the CFW report decoder (settings
|
||||
* field 105), shared by ambient-light.ts and ambient-light.ios.ts.
|
||||
*/
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
|
||||
/** The options with their defaults filled in. */
|
||||
export function resolvePollingOptions(options: AmbientLightPollingOptions): Required<AmbientLightPollingOptions> {
|
||||
return {
|
||||
intervalMs: options.intervalMs ?? 500,
|
||||
minDelta: options.minDelta ?? 1,
|
||||
heartbeatMs: options.heartbeatMs ?? 5000,
|
||||
bindToLease: options.bindToLease ?? true,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
/**
|
||||
* Ambient light sensor access on iOS: the same API as ambient-light.ts (see
|
||||
* there for the CFW query and passive-polling semantics) over the running
|
||||
* shared-Kotlin session. No-ops while no session is running.
|
||||
*/
|
||||
|
||||
/** Ask the firmware for one ambient-light report (delivered to listeners). */
|
||||
export function queryAmbientLight(): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.queryAmbientLight();
|
||||
} catch (error) {
|
||||
console.warn(`queryAmbientLight failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Start or stop CFW passive light-sensor polling. */
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
|
||||
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addAmbientLightListener((body) => {
|
||||
const report = decodeAmbientLightReport(body);
|
||||
if (report) listener(report);
|
||||
});
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -16,88 +16,17 @@
|
||||
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
|
||||
*/
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -123,14 +52,9 @@ export function queryAmbientLight(): void {
|
||||
* on every session start.
|
||||
*/
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
activeCommunicator()?.setAmbientLightPolling(
|
||||
enabled,
|
||||
options.intervalMs ?? 500,
|
||||
options.minDelta ?? 1,
|
||||
options.heartbeatMs ?? 5000,
|
||||
options.bindToLease ?? true,
|
||||
);
|
||||
activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/**
|
||||
* Passive view of the glasses' audio traffic on iOS: the same API as
|
||||
* audio-monitor.ts over the running shared-Kotlin session. Nothing here sends
|
||||
* anything to the glasses.
|
||||
*/
|
||||
|
||||
export type AudioArm = "L" | "R" | "?";
|
||||
|
||||
/** The session listeners attach to; changes across reconnects. */
|
||||
export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */
|
||||
export function isFaceclawAudioCaptureActive(): boolean {
|
||||
try {
|
||||
return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe `communicator` to every audio-characteristic packet and every
|
||||
* field-104 mic status record body (undecoded). Delivered on the main thread.
|
||||
* Returns an unsubscribe function.
|
||||
*/
|
||||
export function addAudioMonitorListeners(
|
||||
communicator: FaceclawCommunicatorBridge | null,
|
||||
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
|
||||
onStatus: (body: Uint8Array, arm: AudioArm) => void,
|
||||
): () => void {
|
||||
if (!communicator) return () => {};
|
||||
const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs));
|
||||
const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm)));
|
||||
return () => {
|
||||
try {
|
||||
offPackets();
|
||||
offStatus();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function toArm(arm: string): AudioArm {
|
||||
return arm === "L" || arm === "R" ? arm : "?";
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
export type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
|
||||
// The iOS twin of the Android benchmark bridge. iOS has no static active
|
||||
// communicator, so the preview controller binds its session here on connect
|
||||
// and unbinds it on close; the run itself is the same shared-Kotlin
|
||||
// GlassesSessionCore stream as on Android.
|
||||
type BenchmarkSession = {
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean
|
||||
cancelBandwidthBenchmark(): void
|
||||
getBandwidthBenchmarkStatus(): string
|
||||
}
|
||||
let session: BenchmarkSession | null = null
|
||||
|
||||
export function bindBandwidthBenchmarkSession(next: BenchmarkSession | null): void {
|
||||
session = next
|
||||
}
|
||||
|
||||
/** Start a run. False if not connected or a run is already active. */
|
||||
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
|
||||
if (!session) return false
|
||||
try {
|
||||
return session.startBandwidthBenchmark(
|
||||
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode))
|
||||
} catch (error) {
|
||||
console.warn(`startBandwidthBenchmark failed: ${error}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Cancel an in-progress run. Safe to call when idle. */
|
||||
export function cancelBandwidthBenchmark(): void {
|
||||
try {
|
||||
session?.cancelBandwidthBenchmark()
|
||||
} catch (error) {
|
||||
console.warn(`cancelBandwidthBenchmark failed: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Status of the current or most recent run; null when not connected. */
|
||||
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
|
||||
if (!session) return null
|
||||
try {
|
||||
return JSON.parse(session.getBandwidthBenchmarkStatus()) as BandwidthBenchmarkStatus
|
||||
} catch (error) {
|
||||
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,9 @@ export type BandwidthBenchmarkStatus = {
|
||||
payloadBytesAcked: number;
|
||||
wireBytesAcked: number;
|
||||
aborted: boolean;
|
||||
/** Each arm's current PHY ("2M", "1M", ...); null when the platform doesn't report it. */
|
||||
leftPhy?: string | null;
|
||||
rightPhy?: string | null;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
|
||||
@@ -2,11 +2,18 @@ import { Http } from '@nativescript/core';
|
||||
import { withUserAgent } from '../util/http';
|
||||
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
|
||||
|
||||
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
|
||||
/**
|
||||
* Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies.
|
||||
*
|
||||
* toArrayBuffer() is interop.bufferFromData, which in @nativescript/ios 9.0.3
|
||||
* views the NSData's bytes without retaining it: once the response is
|
||||
* collected, the buffer is dangling (a store EHPK was saved as 41 MB of
|
||||
* zeros). Copy while the response is still alive.
|
||||
*/
|
||||
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
|
||||
const response = await Http.request({ url, method: 'GET', headers: withUserAgent(), timeout: timeoutMs });
|
||||
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()).slice(), contentLength: Number(length ?? 0) };
|
||||
}
|
||||
|
||||
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
|
||||
|
||||
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
};
|
||||
|
||||
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
create: (options, selection) => new WhisperServerSttClient(options, {
|
||||
baseUrl: selection.whisperServerUrl ?? "",
|
||||
model: selection.whisperServerModel ?? "",
|
||||
language: selection.whisperServerLanguage ?? "",
|
||||
language: selection.language ?? "",
|
||||
}),
|
||||
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
|
||||
},
|
||||
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture transcribes with: the provider, and for Whisper server the
|
||||
* language it is told to expect. A running capture is shared only by holders
|
||||
* asking for the same.
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
|
||||
return selection.provider === "whisper-server"
|
||||
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
|
||||
: selection.provider;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
@@ -105,7 +106,7 @@ export function createCloudSttClient(
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => provider.create(clientOptions, selection),
|
||||
{ ...options, apiKey },
|
||||
{ ...options, apiKey, language: (selection.language ?? "").trim() },
|
||||
continuous,
|
||||
provider.finalTimeoutMs,
|
||||
);
|
||||
|
||||
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
|
||||
|
||||
export type CloudSttOptions = {
|
||||
apiKey: string;
|
||||
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
|
||||
language?: string;
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
|
||||
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
// Without language_code the model detects the language, which can go
|
||||
// badly wrong on short or noisy segments (Japanese heard as Russian).
|
||||
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input";
|
||||
import { File, knownFolders, path, type ImageSource } from "@nativescript/core";
|
||||
import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat";
|
||||
import { type CompassEvent } from "./compass-types";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
import { fromData, toData } from "./kotlin-data";
|
||||
import { previewPixels } from "./ios-graphics";
|
||||
import { iosBluetooth } from "./ios-bluetooth";
|
||||
@@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any;
|
||||
declare const FaceclawKitIosAncsListener: any;
|
||||
declare const FaceclawKitIosAudioPacketListener: any;
|
||||
declare const FaceclawKitFaceclawCompassListener: any;
|
||||
declare const FaceclawKitFaceclawImuListener: any;
|
||||
declare const FaceclawKitIosAmbientLightListener: any;
|
||||
declare const FaceclawKitIosMicStatusListener: any;
|
||||
|
||||
|
||||
export type CommunicatorPhase =
|
||||
@@ -164,6 +168,29 @@ class CompassListener extends NSObject {
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class ImuListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitFaceclawImuListener];
|
||||
handler!: (reading: ImuReading) => void;
|
||||
onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void {
|
||||
this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) });
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class AmbientLightListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosAmbientLightListener];
|
||||
handler!: (body: Uint8Array) => void;
|
||||
onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); }
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class MicStatusListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosMicStatusListener];
|
||||
handler!: (body: Uint8Array, arm: string) => void;
|
||||
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS twin of the Android communicator bridge: the same public API over
|
||||
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
|
||||
@@ -173,6 +200,8 @@ class CompassListener extends NSObject {
|
||||
* Android so callers never re-enter the bridge from a Kotlin frame.
|
||||
*/
|
||||
export class FaceclawCommunicatorBridge {
|
||||
// The started, not yet closed bridge (see getActive).
|
||||
private static active: FaceclawCommunicatorBridge | null = null;
|
||||
private readonly communicator: any;
|
||||
private readonly listenerProxy: SessionListener;
|
||||
private ancsProxy: AncsListener | null = null;
|
||||
@@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge {
|
||||
private audioListener: AudioPacketListener | null = null;
|
||||
private microphoneWork: Promise<void> = Promise.resolve();
|
||||
private microphoneToken = 0;
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* The bridge whose session is running, for modules that reach the glasses
|
||||
* without the bridge threaded through (imu, ambient-light, audio-monitor):
|
||||
* the iOS stand-in for Android's FaceclawBleCommunicator.getActive().
|
||||
* Set by start(), cleared by close().
|
||||
*/
|
||||
static getActive(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.active;
|
||||
}
|
||||
|
||||
constructor(addresses: { right: string; left: string; ring?: string; ringRole?: string }) {
|
||||
this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress(
|
||||
@@ -454,7 +494,9 @@ export class FaceclawCommunicatorBridge {
|
||||
// ----- session ---------------------------------------------------------------------
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.start(); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this;
|
||||
});
|
||||
}
|
||||
|
||||
async setG2ScreenOn(screenOn: boolean): Promise<void> {
|
||||
@@ -644,6 +686,8 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.closed = true;
|
||||
if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null;
|
||||
this.setAncsListeners(false);
|
||||
this.previewAnimationListeners.clear();
|
||||
await this.enqueueNativeCall(() => {
|
||||
@@ -667,6 +711,85 @@ export class FaceclawCommunicatorBridge {
|
||||
return () => { this.communicator.removeCompassListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Developer > Debug tests > BLE bandwidth; see app/native/bandwidth-benchmark.ios.ts. */
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean {
|
||||
return Boolean(this.communicator.startBandwidthBenchmarkWithLinkModeMessageSizeWindowSizeDurationMsLinkMode(
|
||||
messageSize, windowSize, durationMs, linkMode));
|
||||
}
|
||||
|
||||
cancelBandwidthBenchmark(): void {
|
||||
this.communicator.cancelBandwidthBenchmark();
|
||||
}
|
||||
|
||||
/** Status/results of the current or most recent run, as JSON. */
|
||||
getBandwidthBenchmarkStatus(): string {
|
||||
return String(this.communicator.getBandwidthBenchmarkStatus());
|
||||
}
|
||||
|
||||
// ----- sensors and passive audio, for the developer demos -----------------------------
|
||||
// Fire-and-forget, so they skip the native-call queue like the compass controls do.
|
||||
|
||||
/** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */
|
||||
setImuReportEnabled(enable: boolean, reportFrq: number): void {
|
||||
this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq)));
|
||||
}
|
||||
|
||||
/** Accelerometer samples, delivered on the main queue. */
|
||||
addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const proxy = ImuListener.new() as ImuListener;
|
||||
proxy.handler = listener;
|
||||
this.communicator.addImuListenerListener(proxy);
|
||||
return () => { this.communicator.removeImuListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */
|
||||
queryAmbientLight(): void {
|
||||
this.communicator.queryAmbientLight();
|
||||
}
|
||||
|
||||
setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void {
|
||||
this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable),
|
||||
Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease));
|
||||
}
|
||||
|
||||
/** Raw field-105 ambient-light records, delivered on the main queue. */
|
||||
addAmbientLightListener(listener: (body: Uint8Array) => void): () => void {
|
||||
const proxy = AmbientLightListener.new() as AmbientLightListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAmbientLightListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */
|
||||
addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const proxy = MicStatusListener.new() as MicStatusListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addMicStatusListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeMicStatusListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */
|
||||
sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean {
|
||||
return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple(
|
||||
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every audio-characteristic packet, whatever enabled the mics, on the
|
||||
* main queue. Passive: sends nothing and doesn't affect capture.
|
||||
*/
|
||||
addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void {
|
||||
const proxy = AudioPacketListener.new() as AudioPacketListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAudioMonitorListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */
|
||||
isAudioCaptureActive(): boolean {
|
||||
return Boolean(this.communicator.isAudioCaptureActive());
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialized glasses microphone enable/disable (the voice bridge's
|
||||
* IosMicrophoneSession contract). A release while an enable awaits its ACK
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
/**
|
||||
* IMU (accelerometer) access on iOS: the same API as imu.ts over the running
|
||||
* shared-Kotlin session. As there, the stream is experimental, the calls are
|
||||
* no-ops while no session is running, and a listener stays with the session
|
||||
* it subscribed to.
|
||||
*/
|
||||
|
||||
/** Enable/disable the accelerometer report stream. No-op if not connected. */
|
||||
export function setImuReportEnabled(enable: boolean, reportFreq: number): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq);
|
||||
} catch (error) {
|
||||
console.warn(`setImuReportEnabled failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to IMU readings; returns an unsubscribe function. */
|
||||
export function addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addImuListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may be gone after a disconnect; nothing to clean up.
|
||||
}
|
||||
};
|
||||
}
|
||||
+3
-15
@@ -12,7 +12,9 @@ declare const com: any;
|
||||
* static, so it works from any isolate without threading the bridge through.
|
||||
*/
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||
|
||||
export type LocalModelState = {
|
||||
/**
|
||||
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||
*/
|
||||
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||
detail?: string;
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const FaceclawSystemLanguageModel: any;
|
||||
|
||||
/**
|
||||
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||
*
|
||||
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||
* system model runs out of process on the Neural Engine, needs no download,
|
||||
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||
* and tool definitions into it), Apple's guardrails, a background request
|
||||
* budget, and Apple Intelligence-capable hardware.
|
||||
*
|
||||
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||
* parks each call and reports it; here that ends the loop iteration with
|
||||
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||
* provider, and its next request (ending in the matching tool_result blocks)
|
||||
* resumes the parked generation instead of starting a new one.
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "apple-foundation-model",
|
||||
label: "Apple Intelligence",
|
||||
sizeBytes: 0,
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
/**
|
||||
* Only used when tool results arrive with no parked generation to resume (it
|
||||
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||
* continue after a tool output without a new prompt.
|
||||
*/
|
||||
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||
|
||||
type Availability =
|
||||
| "available"
|
||||
| "unsupportedOS"
|
||||
| "deviceNotEligible"
|
||||
| "appleIntelligenceNotEnabled"
|
||||
| "modelNotReady"
|
||||
| "unavailable";
|
||||
|
||||
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "needs iOS 26",
|
||||
deviceNotEligible: "not on this iPhone",
|
||||
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||
modelNotReady: "downloading",
|
||||
unavailable: "unavailable",
|
||||
};
|
||||
|
||||
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||
appleIntelligenceNotEnabled:
|
||||
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||
};
|
||||
|
||||
export function localModelAvailability(): Availability {
|
||||
try {
|
||||
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||
const value = String(FaceclawSystemLanguageModel.availability());
|
||||
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||
} catch {
|
||||
return "unsupportedOS";
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
const availability = localModelAvailability();
|
||||
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
return localModelAvailability() === "available";
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
const availability = localModelAvailability();
|
||||
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||
}
|
||||
|
||||
/** The user-facing reason the model can't run, or "" when it can. */
|
||||
export function localModelUnavailableMessage(): string {
|
||||
const availability = localModelAvailability();
|
||||
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||
}
|
||||
|
||||
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||
// delete or unload, and no progress to watch.
|
||||
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||
return () => {};
|
||||
}
|
||||
export function startLocalModelDownload(): void {}
|
||||
export function cancelLocalModelDownload(): void {}
|
||||
export function deleteLocalModel(): void {}
|
||||
export function unloadLocalModel(): void {}
|
||||
|
||||
type HistoryEntry =
|
||||
| { type: "prompt" | "response"; text: string }
|
||||
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||
|
||||
export type SystemModelRequest = {
|
||||
system: string;
|
||||
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||
history: HistoryEntry[];
|
||||
prompt: string;
|
||||
maxTokens: number;
|
||||
permissive: boolean;
|
||||
};
|
||||
|
||||
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||
|
||||
/**
|
||||
* Translate provider-neutral history into the Swift request: everything but
|
||||
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||
* the model answers.
|
||||
*/
|
||||
export function buildSystemModelRequest(
|
||||
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||
): SystemModelRequest {
|
||||
const messages = options.messages;
|
||||
const last = messages[messages.length - 1];
|
||||
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||
const history: HistoryEntry[] = [];
|
||||
const toolNames = new Map<string, string>();
|
||||
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||
appendHistory(history, message, toolNames);
|
||||
}
|
||||
const tools = (options.tools ?? []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
}));
|
||||
return {
|
||||
system: options.system?.trim() || "You are a helpful assistant.",
|
||||
tools,
|
||||
history,
|
||||
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||
permissive: tools.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||
if (message.role === "user") {
|
||||
for (const result of toolResultsOf(message)) {
|
||||
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "prompt", text });
|
||||
return;
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "response", text });
|
||||
if (typeof message.content === "string") return;
|
||||
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||
for (const block of message.content) {
|
||||
if (block.type !== "tool_use") continue;
|
||||
toolNames.set(block.id, block.name);
|
||||
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||
}
|
||||
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||
}
|
||||
|
||||
function textOf(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content.trim();
|
||||
return content
|
||||
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||
}
|
||||
|
||||
function parseArguments(json: string): any {
|
||||
try {
|
||||
const value = JSON.parse(json);
|
||||
return value && typeof value === "object" ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
type NativeEvent = {
|
||||
kind: string;
|
||||
text?: string;
|
||||
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Generations parked in tool calls, by call id. */
|
||||
const parked = new Map<string, Generation>();
|
||||
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||
const live = new Set<Generation>();
|
||||
|
||||
/**
|
||||
* One native generation, possibly spanning several loop iterations: each
|
||||
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||
* next one resumes it with the results.
|
||||
*/
|
||||
class Generation {
|
||||
private readonly native: any;
|
||||
private options: LlmStreamOptions | null;
|
||||
private iteration = 0;
|
||||
private text = "";
|
||||
private callIds: string[] = [];
|
||||
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||
private closed = false;
|
||||
|
||||
constructor(options: LlmStreamOptions) {
|
||||
this.options = options;
|
||||
this.native = FaceclawSystemLanguageModel.new();
|
||||
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||
live.add(this);
|
||||
}
|
||||
|
||||
start(request: SystemModelRequest): LlmStreamHandle {
|
||||
this.native.start(JSON.stringify(request));
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
/** Whether these results answer this generation's parked calls. */
|
||||
awaits(results: ToolResultBlock[]): boolean {
|
||||
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||
}
|
||||
|
||||
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||
this.unpark();
|
||||
this.options = options;
|
||||
this.iteration++;
|
||||
this.text = "";
|
||||
this.native.resolveToolCalls(
|
||||
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||
);
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
try {
|
||||
this.native.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private handle(): LlmStreamHandle {
|
||||
const iteration = this.iteration;
|
||||
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||
// since then: its owner gave up on the turn.
|
||||
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||
}
|
||||
|
||||
private close(): void {
|
||||
this.closed = true;
|
||||
this.unpark();
|
||||
this.options = null;
|
||||
this.native.eventHandler = null;
|
||||
live.delete(this);
|
||||
}
|
||||
|
||||
private park(ids: string[]): void {
|
||||
this.callIds = ids;
|
||||
for (const id of ids) parked.set(id, this);
|
||||
this.expiry = setTimeout(() => {
|
||||
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||
this.cancel();
|
||||
}, PARKED_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private unpark(): void {
|
||||
for (const id of this.callIds) {
|
||||
if (parked.get(id) === this) parked.delete(id);
|
||||
}
|
||||
this.callIds = [];
|
||||
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||
this.expiry = null;
|
||||
}
|
||||
|
||||
private onEvent(json: string): void {
|
||||
if (this.closed) return;
|
||||
let event: NativeEvent;
|
||||
try {
|
||||
event = JSON.parse(json);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "log") {
|
||||
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||
return;
|
||||
}
|
||||
const options = this.options;
|
||||
if (!options) return;
|
||||
switch (event.kind) {
|
||||
case "delta": {
|
||||
const delta = event.text ?? "";
|
||||
this.text += delta;
|
||||
options.onTextDelta?.(delta, this.text);
|
||||
return;
|
||||
}
|
||||
case "toolCalls": {
|
||||
const calls = event.calls ?? [];
|
||||
const text = this.text.trim();
|
||||
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||
for (const call of calls) {
|
||||
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||
}
|
||||
this.options = null;
|
||||
this.park(calls.map((call) => call.id));
|
||||
options.onDone({ text, content, stopReason: "tool_use" });
|
||||
return;
|
||||
}
|
||||
case "done": {
|
||||
this.close();
|
||||
const text = this.text.trim();
|
||||
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||
return;
|
||||
}
|
||||
case "error": {
|
||||
this.close();
|
||||
if (event.code === "refusal") {
|
||||
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||
} else {
|
||||
options.onError(event.message || "The on-device model failed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a reply from the system model. A request whose history ends in tool
|
||||
* results for a parked generation resumes it; anything else starts afresh.
|
||||
*/
|
||||
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||
const availability = localModelAvailability();
|
||||
if (availability !== "available") {
|
||||
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||
let generation: Generation | null = null;
|
||||
try {
|
||||
generation = new Generation(options);
|
||||
return generation.start(buildSystemModelRequest(options));
|
||||
} catch (error) {
|
||||
generation?.cancel();
|
||||
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The on-phone LLM: one entry point for every feature that can run on the
|
||||
* phone without an API key (the direct assistant's "local" provider,
|
||||
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||
* contract, plus model status for Settings.
|
||||
*
|
||||
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||
*/
|
||||
export {
|
||||
LOCAL_MODEL,
|
||||
isLocalModelReady,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
unloadLocalModel,
|
||||
streamLocalQwen as streamLocalModel,
|
||||
} from "./llama";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -6,6 +6,9 @@ import {
|
||||
type MediaBrowserApp,
|
||||
type MediaBrowserBridge,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify.ios";
|
||||
import type { SpotifyLibrary } from "./spotify-library";
|
||||
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
|
||||
|
||||
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
|
||||
|
||||
@@ -30,7 +33,7 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS"));
|
||||
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Not an Apple Music library"));
|
||||
if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed"));
|
||||
this.connected = true;
|
||||
return Promise.resolve("root");
|
||||
@@ -67,4 +70,42 @@ export class AppleMusicBrowserBridge implements MediaBrowserBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge();
|
||||
/** Apple Music's library, plus the Spotify library once signed in. */
|
||||
export class IosMediaBrowserBridge implements MediaBrowserBridge {
|
||||
private current: MediaBrowserBridge | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly appleMusic: AppleMusicBrowserBridge,
|
||||
private readonly spotify: () => SpotifyLibrary | null,
|
||||
) {}
|
||||
|
||||
listBrowsableApps(): MediaBrowserApp[] {
|
||||
return [...this.appleMusic.listBrowsableApps(), ...(this.spotify()?.listBrowsableApps() ?? [])];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
const library = app.packageName === SPOTIFY_IOS_BUNDLE_ID ? this.spotify() : this.appleMusic;
|
||||
if (!library) return Promise.reject(new Error(`${app.appName} can't be browsed`));
|
||||
this.current = library;
|
||||
return library.connect(app);
|
||||
}
|
||||
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
this.current?.playFromMediaId(mediaId);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.current?.disconnect();
|
||||
this.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new IosMediaBrowserBridge(
|
||||
new AppleMusicBrowserBridge(),
|
||||
() => spotifyService()?.library ?? null,
|
||||
);
|
||||
@@ -2,6 +2,9 @@ import { Utils } from "@nativescript/core";
|
||||
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types";
|
||||
import { spotifyService } from "./spotify";
|
||||
import type { SpotifyLibrary } from "./spotify-library";
|
||||
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -157,4 +160,54 @@ export class FaceclawMediaBrowserBridge implements MediaBrowserBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge();
|
||||
/**
|
||||
* The Android libraries: apps' media browser services, with Spotify's
|
||||
* library from its Web API instead. Spotify's own browser service admits
|
||||
* only allowlisted callers (Android Auto, Wear, Assistant), so connecting to
|
||||
* it always fails; signed in, its row opens the Web API library, and signed
|
||||
* out, opening it says how to sign in. Signed in with Spotify not installed,
|
||||
* the library is still offered (it plays on the user's other devices).
|
||||
*/
|
||||
export class AndroidMediaBrowserBridge implements MediaBrowserBridge {
|
||||
private current: MediaBrowserBridge | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly services: MediaBrowserBridge,
|
||||
private readonly spotify: () => SpotifyLibrary | null,
|
||||
private readonly isEnabled: (packageName: string) => boolean = isMediaAppEnabled,
|
||||
) {}
|
||||
|
||||
listBrowsableApps(refresh = false): MediaBrowserApp[] {
|
||||
const apps = this.services.listBrowsableApps(refresh);
|
||||
const library = this.spotify();
|
||||
if (!library || apps.some((app) => app.packageName === SPOTIFY_ANDROID_PACKAGE) || !this.isEnabled(SPOTIFY_ANDROID_PACKAGE)) {
|
||||
return apps;
|
||||
}
|
||||
return [...apps, ...library.listBrowsableApps()];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
const library = app.packageName === SPOTIFY_ANDROID_PACKAGE ? this.spotify() : null;
|
||||
this.current = library ?? this.services;
|
||||
return this.current.connect(library ? library.app : app);
|
||||
}
|
||||
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
return this.current ? this.current.browse(parentId) : Promise.reject(new Error("Not connected"));
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
this.current?.playFromMediaId(mediaId);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.current?.disconnect();
|
||||
this.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new AndroidMediaBrowserBridge(
|
||||
new FaceclawMediaBrowserBridge(),
|
||||
() => spotifyService()?.library ?? null,
|
||||
);
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify.ios";
|
||||
import type { SpotifyPlayer } from "./spotify-player";
|
||||
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
@@ -88,7 +90,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
void this.start();
|
||||
this.stateListeners.add(listener);
|
||||
this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS);
|
||||
this.pollTimer ??= setInterval(() => this.publish(), POLL_MS);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.stateListeners.delete(listener);
|
||||
@@ -142,7 +144,7 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
if (this.hasAccess()) return Promise.resolve(true);
|
||||
this.pendingAccess ??= this.askForAccess().finally(() => {
|
||||
this.pendingAccess = null;
|
||||
this.refresh();
|
||||
this.publish();
|
||||
});
|
||||
return this.pendingAccess;
|
||||
}
|
||||
@@ -187,12 +189,12 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
const music = FaceclawAppleMusic.shared();
|
||||
if (!this.bound) {
|
||||
this.bound = true;
|
||||
music.changeHandler = () => this.refresh();
|
||||
music.changeHandler = () => this.publish();
|
||||
}
|
||||
return music;
|
||||
}
|
||||
|
||||
private refresh(): void {
|
||||
private publish(): void {
|
||||
if (!this.stateListeners.size) return;
|
||||
const state = this.snapshot();
|
||||
const signature = stateSignature(state);
|
||||
@@ -228,4 +230,192 @@ export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge();
|
||||
export type IosMediaSource = "apple-music" | "spotify";
|
||||
|
||||
/**
|
||||
* Which player to follow: the one playing; with both playing, the current
|
||||
* choice (no flapping); with neither, the one that played last, and before
|
||||
* either has played this run, Spotify when it reports a session (its player
|
||||
* reports only a recently active one, while the system Music player keeps
|
||||
* its last item for days).
|
||||
*/
|
||||
export function chooseMediaSource(
|
||||
current: IosMediaSource,
|
||||
appleMusic: MediaControllerState,
|
||||
spotify: MediaControllerState | null,
|
||||
appleMusicLastPlayingAtMs: number,
|
||||
spotifyLastPlayingAtMs: number,
|
||||
): IosMediaSource {
|
||||
if (!spotify?.accessEnabled) return "apple-music";
|
||||
if (!appleMusic.accessEnabled) return "spotify";
|
||||
const applePlaying = appleMusic.playbackState === "playing";
|
||||
const spotifyPlaying = spotify.playbackState === "playing";
|
||||
if (applePlaying && spotifyPlaying) return current;
|
||||
if (applePlaying) return "apple-music";
|
||||
if (spotifyPlaying) return "spotify";
|
||||
if (appleMusicLastPlayingAtMs !== spotifyLastPlayingAtMs) {
|
||||
return spotifyLastPlayingAtMs > appleMusicLastPlayingAtMs ? "spotify" : "apple-music";
|
||||
}
|
||||
return spotify.available ? "spotify" : "apple-music";
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS Music bridge: Apple Music (the system player), plus Spotify once
|
||||
* signed in (its Web API: playback on this phone or any Spotify Connect
|
||||
* device). The state and every command follow one of them, by
|
||||
* chooseMediaSource.
|
||||
*/
|
||||
export class IosMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt =
|
||||
"Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone. For Spotify, sign in from the app menu instead.";
|
||||
readonly accessLabel: string;
|
||||
readonly accessAction: string;
|
||||
private source: IosMediaSource = "apple-music";
|
||||
private appleMusicLastPlayingAtMs = 0;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private unsubscribeSources: Array<() => void> = [];
|
||||
/** The source and its artVersion the composite artVersion was bumped for. */
|
||||
private artKey = "";
|
||||
private artVersion = 0;
|
||||
|
||||
constructor(
|
||||
private readonly appleMusic: AppleMusicControllerBridge,
|
||||
private readonly spotify: () => SpotifyPlayer | null,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {
|
||||
this.accessLabel = appleMusic.accessLabel;
|
||||
this.accessAction = appleMusic.accessAction;
|
||||
}
|
||||
|
||||
get skipRebuildsQueue(): boolean {
|
||||
return this.active().skipRebuildsQueue;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
if (!this.listeners.size) {
|
||||
const update = () => this.emit();
|
||||
const spotify = this.spotify();
|
||||
this.unsubscribeSources = [this.appleMusic.onStateChange(update), ...(spotify ? [spotify.onStateChange(update)] : [])];
|
||||
}
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (this.listeners.size) return;
|
||||
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
|
||||
this.unsubscribeSources = [];
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const appleMusic = this.appleMusic.snapshot();
|
||||
const spotify = this.spotify()?.snapshot() ?? null;
|
||||
this.select(appleMusic, spotify);
|
||||
const state = this.source === "spotify" && spotify ? spotify : appleMusic;
|
||||
// Changes with the source as well as with that source's art.
|
||||
const artKey = `${this.source}:${state.artVersion}`;
|
||||
if (artKey !== this.artKey) {
|
||||
this.artKey = artKey;
|
||||
this.artVersion++;
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
status: state.available ? state.status : idleStatus(appleMusic, spotify, state),
|
||||
artVersion: this.artVersion,
|
||||
// A Spotify command can fail while Apple Music is the one shown (playing from the Spotify library).
|
||||
notice: spotify?.notice || state.notice || "",
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.appleMusic.start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.appleMusic.stop();
|
||||
}
|
||||
|
||||
playPause(): Promise<void> {
|
||||
return this.active().playPause();
|
||||
}
|
||||
|
||||
skipNext(): Promise<void> {
|
||||
return this.active().skipNext();
|
||||
}
|
||||
|
||||
skipPrevious(): Promise<void> {
|
||||
return this.active().skipPrevious();
|
||||
}
|
||||
|
||||
/** Apple Music's media-library permission (what requestAccess asks for); Spotify signs in from the Music menu. */
|
||||
hasAccess(): boolean {
|
||||
return this.appleMusic.hasAccess();
|
||||
}
|
||||
|
||||
requestAccess(): Promise<boolean> {
|
||||
return this.appleMusic.requestAccess();
|
||||
}
|
||||
|
||||
skipToQueueItem(id: number): Promise<void> {
|
||||
return this.active().skipToQueueItem(id);
|
||||
}
|
||||
|
||||
getMediaVolumePercent(): number {
|
||||
return this.active().getMediaVolumePercent();
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
this.active().setMediaVolumePercent(volumePercent);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
return this.active().getAlbumArt(maxSize);
|
||||
}
|
||||
|
||||
getQueue(): MediaQueueItem[] {
|
||||
return this.active().getQueue();
|
||||
}
|
||||
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
await this.active().sendCustomAction?.(action);
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.spotify()?.refresh();
|
||||
}
|
||||
|
||||
private active(): MediaControllerBridge {
|
||||
const spotify = this.spotify();
|
||||
this.select(this.appleMusic.snapshot(), spotify?.snapshot() ?? null);
|
||||
return this.source === "spotify" && spotify ? spotify : this.appleMusic;
|
||||
}
|
||||
|
||||
private select(appleMusic: MediaControllerState, spotify: MediaControllerState | null): void {
|
||||
if (appleMusic.playbackState === "playing") this.appleMusicLastPlayingAtMs = this.now();
|
||||
this.source = chooseMediaSource(this.source, appleMusic, spotify, this.appleMusicLastPlayingAtMs,
|
||||
this.spotify()?.lastPlayingAtMs ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward every source update: each source already drops repeats, and
|
||||
* Spotify also signals changes that are not in the state (its queue).
|
||||
*/
|
||||
private emit(): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/** The no-session line: which players were checked. */
|
||||
function idleStatus(appleMusic: MediaControllerState, spotify: MediaControllerState | null, shown: MediaControllerState): string {
|
||||
if (appleMusic.accessEnabled && spotify?.accessEnabled) return "Nothing is playing.";
|
||||
return shown.status;
|
||||
}
|
||||
|
||||
export const appleMusicControllerBridge = new AppleMusicControllerBridge();
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new IosMediaControllerBridge(
|
||||
appleMusicControllerBridge,
|
||||
() => spotifyService()?.player ?? null,
|
||||
);
|
||||
@@ -12,11 +12,14 @@ import {
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import { spotifyService } from "./spotify";
|
||||
import { SPOTIFY_ANDROID_PACKAGE, type SpotifyPlayer } from "./spotify-player";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
/** Android media sessions (any player that publishes one), through FaceclawMediaController. */
|
||||
export class FaceclawMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings.";
|
||||
readonly accessLabel = "Notification access needed";
|
||||
@@ -215,4 +218,251 @@ function normalizePlaybackState(value: string): MediaPlaybackState {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge();
|
||||
export type AndroidMediaSource = "session" | "spotify";
|
||||
|
||||
/**
|
||||
* Which to follow: the media session (any player, Spotify included when it
|
||||
* runs on this phone) or Spotify's Web API player (signed in; it also sees
|
||||
* Spotify playing on a computer or speaker). The one playing; with both
|
||||
* playing, the session when it is Spotify's own (the same playback), else the
|
||||
* current choice (no flapping); with neither, a Spotify session, then the one
|
||||
* that played last, then whichever has something. Without notification
|
||||
* access the session shows its access prompt unless Spotify has something.
|
||||
*/
|
||||
export function chooseAndroidMediaSource(
|
||||
current: AndroidMediaSource,
|
||||
session: MediaControllerState,
|
||||
spotify: MediaControllerState | null,
|
||||
sessionLastPlayingAtMs: number,
|
||||
spotifyLastPlayingAtMs: number,
|
||||
): AndroidMediaSource {
|
||||
if (!spotify?.accessEnabled) return "session";
|
||||
if (!session.accessEnabled) return spotify.available ? "spotify" : "session";
|
||||
const sessionIsSpotify = session.packageName === SPOTIFY_ANDROID_PACKAGE;
|
||||
const sessionPlaying = session.playbackState === "playing";
|
||||
const spotifyPlaying = spotify.playbackState === "playing";
|
||||
if (sessionPlaying && spotifyPlaying) return sessionIsSpotify ? "session" : current;
|
||||
if (sessionPlaying) return "session";
|
||||
if (spotifyPlaying) return "spotify";
|
||||
if (sessionIsSpotify && session.available) return "session";
|
||||
if (sessionLastPlayingAtMs !== spotifyLastPlayingAtMs) {
|
||||
return spotifyLastPlayingAtMs > sessionLastPlayingAtMs ? "spotify" : "session";
|
||||
}
|
||||
return session.available || !spotify.available ? "session" : "spotify";
|
||||
}
|
||||
|
||||
/**
|
||||
* The Android Music bridge: media sessions, plus Spotify's Web API once
|
||||
* signed in (optional). Spotify playing on this phone stays the session's:
|
||||
* its state, art, queue and transport come from the session (instant, no
|
||||
* quota), and the Web API adds what the session lacks: shuffle / repeat /
|
||||
* like rows and the volume of a Spotify Connect speaker or computer. Spotify
|
||||
* playing on another device, with no session here, follows the Web API player.
|
||||
*/
|
||||
export class AndroidMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt: string;
|
||||
readonly accessLabel: string;
|
||||
readonly accessAction: string;
|
||||
private source: AndroidMediaSource = "session";
|
||||
private sessionLastPlayingAtMs = 0;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private unsubscribeSources: Array<() => void> = [];
|
||||
/** The source and its artVersion the composite artVersion was bumped for. */
|
||||
private artKey = "";
|
||||
private artVersion = 0;
|
||||
/** The player the last getQueue listed, so a jump goes to the same one. */
|
||||
private queueSource: MediaControllerBridge | null = null;
|
||||
/** A Spotify session's item and play state, to re-read the Web API when they change. */
|
||||
private spotifySessionKey = "";
|
||||
|
||||
constructor(
|
||||
private readonly session: MediaControllerBridge,
|
||||
private readonly spotify: () => SpotifyPlayer | null,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {
|
||||
this.accessPrompt = session.accessPrompt;
|
||||
this.accessLabel = session.accessLabel;
|
||||
this.accessAction = session.accessAction;
|
||||
}
|
||||
|
||||
get skipRebuildsQueue(): boolean {
|
||||
return (this.queueSource ?? this.active()).skipRebuildsQueue;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
if (!this.listeners.size) {
|
||||
const spotify = this.spotify();
|
||||
this.unsubscribeSources = [
|
||||
this.session.onStateChange((state) => this.sessionChanged(state)),
|
||||
...(spotify ? [spotify.onStateChange(() => this.emit())] : []),
|
||||
];
|
||||
}
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (this.listeners.size) return;
|
||||
for (const unsubscribe of this.unsubscribeSources) unsubscribe();
|
||||
this.unsubscribeSources = [];
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const session = this.session.snapshot();
|
||||
const spotify = this.spotify()?.snapshot() ?? null;
|
||||
this.select(session, spotify);
|
||||
const state = this.source === "spotify" && spotify ? spotify : withSpotifyExtras(session, spotify);
|
||||
// Changes with the source as well as with that source's art.
|
||||
const artKey = `${this.source}:${state.artVersion}`;
|
||||
if (artKey !== this.artKey) {
|
||||
this.artKey = artKey;
|
||||
this.artVersion++;
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
artVersion: this.artVersion,
|
||||
// A Spotify command can fail while the session is the one shown (playing from the Spotify library).
|
||||
notice: spotify?.notice || state.notice || "",
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.session.start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.session.stop();
|
||||
}
|
||||
|
||||
playPause(): Promise<void> {
|
||||
return this.active().playPause();
|
||||
}
|
||||
|
||||
skipNext(): Promise<void> {
|
||||
return this.active().skipNext();
|
||||
}
|
||||
|
||||
skipPrevious(): Promise<void> {
|
||||
return this.active().skipPrevious();
|
||||
}
|
||||
|
||||
/** Notification access (what requestAccess opens); Spotify signs in from the Music menu. */
|
||||
hasAccess(): boolean {
|
||||
return this.session.hasAccess();
|
||||
}
|
||||
|
||||
requestAccess(): Promise<boolean> {
|
||||
return this.session.requestAccess();
|
||||
}
|
||||
|
||||
skipToQueueItem(id: number): Promise<void> {
|
||||
return (this.queueSource ?? this.active()).skipToQueueItem(id);
|
||||
}
|
||||
|
||||
/** Spotify's own volume logic (a Connect device's volume, else this phone's) whenever Spotify is shown. */
|
||||
getMediaVolumePercent(): number {
|
||||
return this.volumeSource().getMediaVolumePercent();
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
this.volumeSource().setMediaVolumePercent(volumePercent);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
return this.active().getAlbumArt(maxSize);
|
||||
}
|
||||
|
||||
/** The session's queue; for a Spotify session that lists none, the Web API's. */
|
||||
getQueue(): MediaQueueItem[] {
|
||||
const active = this.active();
|
||||
const queue = active.getQueue();
|
||||
const spotify = this.spotifyExtras();
|
||||
if (queue.length || active !== this.session || !spotify) {
|
||||
this.queueSource = active;
|
||||
return queue;
|
||||
}
|
||||
this.queueSource = spotify;
|
||||
return spotify.getQueue();
|
||||
}
|
||||
|
||||
/** Spotify's actions (shown with a Spotify session too) go to the Web API player; any others to the session. */
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
const spotify = this.spotify();
|
||||
if (spotify && spotify.snapshot().customActions?.some((custom) => custom.action === action)) {
|
||||
await spotify.sendCustomAction(action);
|
||||
} else {
|
||||
await this.session.sendCustomAction?.(action);
|
||||
}
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.spotify()?.refresh();
|
||||
}
|
||||
|
||||
private active(): MediaControllerBridge {
|
||||
const spotify = this.spotify();
|
||||
this.select(this.session.snapshot(), spotify?.snapshot() ?? null);
|
||||
return this.source === "spotify" && spotify ? spotify : this.session;
|
||||
}
|
||||
|
||||
/** The signed-in Web API player when it describes what the session shows (Spotify's own session); else null. */
|
||||
private spotifyExtras(): SpotifyPlayer | null {
|
||||
const spotify = this.spotify();
|
||||
if (!spotify || this.session.snapshot().packageName !== SPOTIFY_ANDROID_PACKAGE) return null;
|
||||
return spotify.snapshot().available ? spotify : null;
|
||||
}
|
||||
|
||||
private volumeSource(): MediaControllerBridge {
|
||||
const active = this.active();
|
||||
return active === this.session ? this.spotifyExtras() ?? this.session : active;
|
||||
}
|
||||
|
||||
private select(session: MediaControllerState, spotify: MediaControllerState | null): void {
|
||||
if (session.playbackState === "playing") this.sessionLastPlayingAtMs = this.now();
|
||||
this.source = chooseAndroidMediaSource(this.source, session, spotify, this.sessionLastPlayingAtMs,
|
||||
this.spotify()?.lastPlayingAtMs ?? 0);
|
||||
}
|
||||
|
||||
private sessionChanged(state: MediaControllerState): void {
|
||||
// Spotify's session changed track or started / stopped: re-read the Web
|
||||
// API now, so its rows (like this song, shuffle) follow at once rather
|
||||
// than at the next poll. Signed out, the player skips the request.
|
||||
const key = state.packageName === SPOTIFY_ANDROID_PACKAGE ? `${state.title}\n${state.playbackState}` : "";
|
||||
if (key !== this.spotifySessionKey) {
|
||||
this.spotifySessionKey = key;
|
||||
if (key) void this.spotify()?.refresh();
|
||||
}
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/** Forward every source update: each drops its own repeats, and Spotify also signals queue changes. */
|
||||
private emit(): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Spotify's session state plus what only the Web API knows: its custom rows
|
||||
* and, when it plays elsewhere, the device ("Spotify · Kitchen"). "Like
|
||||
* song" names the Web API's current item, so it is left out until both agree
|
||||
* on the track (the API lags a skip by up to a poll).
|
||||
*/
|
||||
export function withSpotifyExtras(session: MediaControllerState, spotify: MediaControllerState | null): MediaControllerState {
|
||||
if (!spotify?.available || session.packageName !== SPOTIFY_ANDROID_PACKAGE) return session;
|
||||
const sameItem = spotify.title === session.title;
|
||||
const actions = (spotify.customActions ?? []).filter((custom) => sameItem || custom.action !== "like");
|
||||
return {
|
||||
...session,
|
||||
appName: sameItem ? spotify.appName : session.appName,
|
||||
customActions: [...(session.customActions ?? []), ...actions],
|
||||
};
|
||||
}
|
||||
|
||||
const sessionBridge = new FaceclawMediaControllerBridge();
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new AndroidMediaControllerBridge(
|
||||
sessionBridge,
|
||||
() => spotifyService()?.player ?? null,
|
||||
);
|
||||
@@ -3,7 +3,8 @@ import type { GrayImage } from "../graphics/image";
|
||||
/**
|
||||
* Shapes shared by the media bridges: media-controller.ts / media-browser.ts
|
||||
* drive Android media sessions and MediaBrowserServices; their .ios.ts
|
||||
* counterparts drive Apple Music through the MediaPlayer framework.
|
||||
* counterparts drive Apple Music through the MediaPlayer framework. Both
|
||||
* add Spotify through its Web API once signed in (spotify-*.ts).
|
||||
*/
|
||||
|
||||
/** The player behind the iOS bridges: the system Music app. */
|
||||
@@ -13,9 +14,19 @@ export const APPLE_MUSIC_PACKAGE = "com.apple.Music";
|
||||
export type MediaQueueItem = {
|
||||
id: number;
|
||||
title: string;
|
||||
/** Usually the artist; absent when the player gives none. */
|
||||
subtitle?: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
/** A player-specific action beyond the transport controls (Spotify's shuffle, repeat, like). */
|
||||
export type MediaCustomAction = {
|
||||
/** Stable id to send back with sendCustomAction. */
|
||||
action: string;
|
||||
/** The row label, naming the current state where there is one ("Shuffle: on"). */
|
||||
name: string;
|
||||
};
|
||||
|
||||
export type MediaPlaybackState =
|
||||
| "notification-access-required"
|
||||
| "idle"
|
||||
@@ -47,6 +58,10 @@ export type MediaControllerState = {
|
||||
* players re-publish the same strings with a different bitmap.
|
||||
*/
|
||||
artVersion: number;
|
||||
/** Player-specific actions (see sendCustomAction); absent when the player offers none. */
|
||||
customActions?: MediaCustomAction[];
|
||||
/** A short-lived message about a command that failed (Spotify needs Premium, ...); absent or "" otherwise. */
|
||||
notice?: string;
|
||||
};
|
||||
|
||||
export const DEFAULT_MEDIA_STATE: MediaControllerState = {
|
||||
@@ -113,6 +128,13 @@ export interface MediaControllerBridge {
|
||||
getAlbumArt(maxSize: number): GrayImage | null;
|
||||
/** The player's queue (playlist), empty when the player exposes none. */
|
||||
getQueue(): MediaQueueItem[];
|
||||
/** Run one of the state's customActions. */
|
||||
sendCustomAction?(action: string): Promise<void>;
|
||||
/**
|
||||
* Re-read the player now, for one-off readers (the assistant) when no
|
||||
* listener keeps the state current; absent when snapshots are always live.
|
||||
*/
|
||||
refresh?(): Promise<void>;
|
||||
}
|
||||
|
||||
export type MediaBrowserApp = {
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/** The iOS twin of mic-control-link.ts, over the running shared-Kotlin session. */
|
||||
|
||||
export function micControlSession(): unknown {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendMicControlRecord(
|
||||
session: unknown,
|
||||
record: Uint8Array,
|
||||
label: string,
|
||||
rightTemple: boolean,
|
||||
leftTemple: boolean,
|
||||
): boolean {
|
||||
return (session as FaceclawCommunicatorBridge).sendFaceclawMicControl(record, label, rightTemple, leftTemple);
|
||||
}
|
||||
|
||||
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
|
||||
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const unsubscribe = (session as FaceclawCommunicatorBridge).addMicStatusListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { toJavaBytes } from "./cloud-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Platform plumbing for the CFW mic_control channel (apps/microphones/mic-control.ts):
|
||||
* the running session to send records on and to hear field-104 status replies
|
||||
* from. The session is opaque and is a different object after a reconnect.
|
||||
*/
|
||||
|
||||
export function micControlSession(): unknown {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendMicControlRecord(
|
||||
session: unknown,
|
||||
record: Uint8Array,
|
||||
label: string,
|
||||
rightTemple: boolean,
|
||||
leftTemple: boolean,
|
||||
): boolean {
|
||||
return (session as any).sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
}
|
||||
|
||||
/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */
|
||||
export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const communicator = session as any;
|
||||
const proxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
||||
onMicStatus: (body: any, arm: string) => listener(toUint8Array(body), String(arm)),
|
||||
});
|
||||
communicator.addMicStatusListener(proxy);
|
||||
return () => {
|
||||
try {
|
||||
communicator.removeMicStatusListener(proxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import type { PebbleIconName } from "../graphics/pebble-icons";
|
||||
|
||||
/**
|
||||
* Icons for notification sources that arrive without one: iPhone
|
||||
* notifications over ANCS carry the app's bundle ID, its display name
|
||||
* (fetched separately) and an ANCS category, but no icon. The icons are
|
||||
* Pebble's (see scripts/generate-pebble-icons.cjs).
|
||||
*
|
||||
* The bundle IDs start from the Pebble firmware's own ANCS table (PebbleOS
|
||||
* include/pbl/services/notifications/ancs/ancs_known_apps.h, minus retired
|
||||
* apps), plus Apple's built-in apps and popular apps mapped to a generic
|
||||
* pictogram. Every App Store bundle ID here was checked against Apple's
|
||||
* iTunes lookup when it was added; Apple's built-in apps follow
|
||||
* github.com/joeblau/apple-bundle-identifiers.
|
||||
*
|
||||
* Display names are a fallback for apps whose bundle ID varies per build:
|
||||
* self-built apps such as Loop sign with their builder's team ID. They match
|
||||
* the whole name, ignoring case, so generic words (an app called "United"
|
||||
* or "Voice") are left out. Bundle prefixes catch regional variants
|
||||
* (com.abbott.librelink.us, .gb, ...).
|
||||
*
|
||||
* Apple's Phone app is deliberately absent: its notifications are calls,
|
||||
* missed calls and voicemail, which the ANCS category tells apart.
|
||||
*/
|
||||
type AppIcon = {
|
||||
icon: PebbleIconName;
|
||||
bundleIds: readonly string[];
|
||||
bundlePrefixes?: readonly string[];
|
||||
names?: readonly string[];
|
||||
};
|
||||
|
||||
const APP_ICONS: readonly AppIcon[] = [
|
||||
// Apple's built-in apps.
|
||||
{ icon: "sms", bundleIds: ["com.apple.MobileSMS"] },
|
||||
{ icon: "email", bundleIds: ["com.apple.mobilemail"] },
|
||||
{ icon: "calendar", bundleIds: ["com.apple.mobilecal"] },
|
||||
{ icon: "reminder", bundleIds: ["com.apple.reminders"] },
|
||||
{ icon: "facetime", bundleIds: ["com.apple.facetime"] },
|
||||
{ icon: "iosPhotos", bundleIds: ["com.apple.mobileslideshow"] },
|
||||
{ icon: "alarmClock", bundleIds: ["com.apple.mobiletimer"] },
|
||||
{ icon: "weather", bundleIds: ["com.apple.weather"] },
|
||||
{ icon: "location", bundleIds: ["com.apple.findmy", "com.apple.Maps"] },
|
||||
{ icon: "settings", bundleIds: ["com.apple.Preferences"] },
|
||||
{ icon: "heart", bundleIds: ["com.apple.Health"] },
|
||||
{ icon: "activity", bundleIds: ["com.apple.Fitness"] },
|
||||
{ icon: "news", bundleIds: ["com.apple.news"] },
|
||||
{ icon: "podcast", bundleIds: ["com.apple.podcasts"] },
|
||||
{ icon: "music", bundleIds: ["com.apple.Music"] },
|
||||
{ icon: "tv", bundleIds: ["com.apple.tv"] },
|
||||
{ icon: "payBill", bundleIds: ["com.apple.Passbook"] },
|
||||
{ icon: "stocks", bundleIds: ["com.apple.stocks"] },
|
||||
{ icon: "appStore", bundleIds: ["com.apple.AppStore"] },
|
||||
{ icon: "appleShortcuts", bundleIds: ["com.apple.shortcuts"] },
|
||||
{ icon: "microphone", bundleIds: ["com.apple.VoiceMemos"] },
|
||||
{ icon: "contacts", bundleIds: ["com.apple.MobileAddressBook"] },
|
||||
{ icon: "shopping", bundleIds: ["com.apple.store.Jolly"] },
|
||||
|
||||
// Apps with an icon of their own.
|
||||
{ icon: "airmail", bundleIds: ["com.airmailapp.iphone"], names: ["Airmail"] },
|
||||
{ icon: "amazon", bundleIds: ["com.amazon.Amazon"], names: ["Amazon"] },
|
||||
{ icon: "bbcNews", bundleIds: ["uk.co.bbc.news"], names: ["BBC News"] },
|
||||
{ icon: "beeper", bundleIds: ["com.automattic.beeper"], names: ["Beeper"] },
|
||||
{ icon: "bluesky", bundleIds: ["xyz.blueskyweb.app"], names: ["Bluesky"] },
|
||||
{ icon: "discord", bundleIds: ["com.hammerandchisel.discord"], names: ["Discord"] },
|
||||
{ icon: "duolingo", bundleIds: ["com.duolingo.DuolingoMobile"], names: ["Duolingo"] },
|
||||
{ icon: "ebay", bundleIds: ["com.ebay.iphone"], names: ["eBay"] },
|
||||
{ icon: "element", bundleIds: ["im.vector.app", "io.element.elementx"], names: ["Element", "Element X", "Element Classic"] },
|
||||
{ icon: "facebook", bundleIds: ["com.facebook.Facebook"], names: ["Facebook"] },
|
||||
{ icon: "gmail", bundleIds: ["com.google.Gmail"], names: ["Gmail"] },
|
||||
{ icon: "googleChat", bundleIds: ["com.google.Dynamite"], names: ["Google Chat"] },
|
||||
{ icon: "googleMaps", bundleIds: ["com.google.Maps"], names: ["Google Maps"] },
|
||||
{ icon: "googlePhotos", bundleIds: ["com.google.photos"], names: ["Google Photos"] },
|
||||
{ icon: "googleTasks", bundleIds: ["com.google.tasks"], names: ["Google Tasks"] },
|
||||
{ icon: "homeAssistant", bundleIds: ["io.robbie.HomeAssistant"], names: ["Home Assistant"] },
|
||||
{ icon: "instagram", bundleIds: ["com.burbn.instagram"], names: ["Instagram"] },
|
||||
{ icon: "kakaoTalk", bundleIds: ["com.iwilab.KakaoTalk"], names: ["KakaoTalk"] },
|
||||
{ icon: "kik", bundleIds: ["com.kik.chat"], names: ["Kik"] },
|
||||
{ icon: "line", bundleIds: ["jp.naver.line"], names: ["LINE"] },
|
||||
{ icon: "linkedIn", bundleIds: ["com.linkedin.LinkedIn"], names: ["LinkedIn"] },
|
||||
{ icon: "messenger", bundleIds: ["com.facebook.Messenger"], names: ["Messenger"] },
|
||||
{ icon: "omniFocus", bundleIds: ["com.omnigroup.OmniFocus4", "com.omnigroup.OmniFocus3.iOS"], names: ["OmniFocus"] },
|
||||
{ icon: "outlook", bundleIds: ["com.microsoft.Office.Outlook"], names: ["Outlook"] },
|
||||
{ icon: "reddit", bundleIds: ["com.reddit.Reddit"], names: ["Reddit"] },
|
||||
{ icon: "signal", bundleIds: ["org.whispersystems.signal"], names: ["Signal"] },
|
||||
{ icon: "simplex", bundleIds: ["chat.simplex.app"], names: ["SimpleX"] },
|
||||
{ icon: "slack", bundleIds: ["com.tinyspeck.chatlyio"], names: ["Slack"] },
|
||||
{ icon: "snapchat", bundleIds: ["com.toyopagroup.picaboo"], names: ["Snapchat"] },
|
||||
{ icon: "steam", bundleIds: ["com.valvesoftware.Steam"], names: ["Steam"] },
|
||||
{ icon: "substack", bundleIds: ["com.substack.Substack"], names: ["Substack"] },
|
||||
{ icon: "swarm", bundleIds: ["com.foursquare.robin"], names: ["Swarm"] },
|
||||
{ icon: "tapo", bundleIds: ["com.tplink.tapo"], names: ["Tapo"] },
|
||||
{ icon: "teams", bundleIds: ["com.microsoft.skype.teams"], names: ["Teams", "Microsoft Teams"] },
|
||||
{ icon: "telegram", bundleIds: ["ph.telegra.Telegraph"], names: ["Telegram"] },
|
||||
{ icon: "threads", bundleIds: ["com.burbn.barcelona"], names: ["Threads"] },
|
||||
{ icon: "tiktok", bundleIds: ["com.zhiliaoapp.musically"], names: ["TikTok"] },
|
||||
{ icon: "twitch", bundleIds: ["tv.twitch"], names: ["Twitch"] },
|
||||
// X, still under Twitter's bundle ID and drawn with its bird, as on Pebble.
|
||||
{ icon: "twitter", bundleIds: ["com.atebits.Tweetie2"], names: ["X", "Twitter"] },
|
||||
{ icon: "unifiProtect", bundleIds: ["com.ubnt.protect"], names: ["UniFi Protect"] },
|
||||
{ icon: "viber", bundleIds: ["com.viber"], names: ["Viber"] },
|
||||
{ icon: "waze", bundleIds: ["com.waze.iphone"], names: ["Waze"] },
|
||||
{ icon: "weChat", bundleIds: ["com.tencent.xin"], names: ["WeChat"] },
|
||||
{ icon: "whatsApp", bundleIds: ["net.whatsapp.WhatsApp"], names: ["WhatsApp"] },
|
||||
{ icon: "whatsAppBusiness", bundleIds: ["net.whatsapp.WhatsAppSMB"], names: ["WA Business", "WhatsApp Business"] },
|
||||
{ icon: "yahooMail", bundleIds: ["com.yahoo.Aerogram"], names: ["Yahoo Mail"] },
|
||||
{ icon: "youtube", bundleIds: ["com.google.ios.youtube"], names: ["YouTube"] },
|
||||
{ icon: "zoom", bundleIds: ["us.zoom.videomeetings"], names: ["Zoom"] },
|
||||
|
||||
// Apps drawn with a generic pictogram.
|
||||
{ icon: "calendar", bundleIds: ["com.google.calendar", "com.flexibits.fantastical2.iphone"], names: ["Google Calendar", "Fantastical"] },
|
||||
{ icon: "email", bundleIds: ["com.readdle.smartemail", "ch.protonmail.protonmail"], names: ["Proton Mail"] },
|
||||
{ icon: "sms", bundleIds: ["com.google.GVDialer", "ch.threema.iapp", "com.groupme.iphone-app"], names: ["Threema", "GroupMe"] },
|
||||
{ icon: "reminder", bundleIds: ["com.todoist.ios", "com.culturedcode.ThingsiPhone", "com.microsoft.to-do"], names: ["Todoist"] },
|
||||
{ icon: "car", bundleIds: ["com.ubercab.UberClient", "com.zimride.instant"], names: ["Uber", "Lyft"] },
|
||||
{ icon: "dinner", bundleIds: ["com.ubercab.UberEats", "doordash.DoorDashConsumer", "com.grubhub.search"], names: ["Uber Eats", "DoorDash", "Grubhub"] },
|
||||
{
|
||||
icon: "glucose",
|
||||
bundleIds: [
|
||||
"com.dexcom.G6", "com.dexcom.g7app", "com.dexcom.dexcomone", "com.dexcom.stelo", "com.dexcom.followglobal",
|
||||
"org.nativescript.LibreLinkUp",
|
||||
],
|
||||
bundlePrefixes: ["com.abbott.librelink.", "com.abbott.libre3."],
|
||||
names: ["Dexcom G6", "Dexcom G7", "Dexcom ONE", "LibreLink", "LibreLinkUp", "Loop", "Trio", "LoopFollow", "xDrip4iOS"],
|
||||
},
|
||||
{ icon: "music", bundleIds: ["com.spotify.client", "com.google.ios.youtubemusic", "com.pandora"], names: ["Spotify", "YouTube Music", "Pandora"] },
|
||||
{ icon: "podcast", bundleIds: ["fm.overcast.overcast", "au.com.shiftyjelly.podcasts"], names: ["Overcast", "Pocket Casts"] },
|
||||
{
|
||||
icon: "tv",
|
||||
bundleIds: ["com.netflix.Netflix", "com.hulu.plus", "com.disney.disneyplus", "com.amazon.aiv.AIVApp"],
|
||||
names: ["Netflix", "Hulu", "Disney+", "Prime Video"],
|
||||
},
|
||||
{
|
||||
icon: "payBill",
|
||||
bundleIds: [
|
||||
"net.kortina.labs.Venmo", "com.squareup.cash", "com.yourcompany.PPClient", "com.chase", "com.bankofamerica.BofA",
|
||||
"com.wf.mobilebanking", "com.capitalone.enterprisemobilebanking",
|
||||
"com.revolut.revolut", "com.transferwise.Transferwise", "de.no26.Number26", "com.bunq.ios",
|
||||
],
|
||||
names: ["Venmo", "Cash App", "PayPal", "Revolut", "N26", "bunq"],
|
||||
},
|
||||
{ icon: "stocks", bundleIds: ["com.robinhood.release.Robinhood", "com.vilcsak.bitcoin2"], names: ["Robinhood", "Coinbase"] },
|
||||
{
|
||||
icon: "flight",
|
||||
bundleIds: [
|
||||
"com.united.UnitedCustomerFacingIPhone", "com.delta.iphone.ver1", "com.aa.AmericanAirlines",
|
||||
"com.southwest.iphoneprod", "com.flightyapp.flighty",
|
||||
],
|
||||
names: ["Fly Delta", "Flighty"],
|
||||
},
|
||||
{ icon: "weather", bundleIds: ["com.weather.TWC", "com.grailr.CARROTweather"] },
|
||||
{ icon: "activity", bundleIds: ["com.strava.stravaride", "com.nike.nikeplus-gps", "com.fitbit.FitbitMobile"], names: ["Strava", "Fitbit"] },
|
||||
{ icon: "heart", bundleIds: ["com.ouraring.oura"], names: ["Oura"] },
|
||||
{ icon: "news", bundleIds: ["com.nytimes.NYTimes", "com.google.GoogleDigitalEditions"], names: ["Google News"] },
|
||||
];
|
||||
|
||||
/**
|
||||
* Icons for the ANCS categories (CategoryID: Other, IncomingCall,
|
||||
* MissedCall, Voicemail, Social, Schedule, Email, News, HealthAndFitness,
|
||||
* BusinessAndFinance, Location, Entertainment), for apps not listed above.
|
||||
*/
|
||||
const CATEGORY_ICONS: readonly PebbleIconName[] = [
|
||||
"generic", "incomingCall", "missedCall", "voicemail", "sms", "calendar",
|
||||
"email", "news", "heart", "stocks", "location", "tv",
|
||||
];
|
||||
|
||||
type Index = { bundleIds: Map<string, PebbleIconName>; prefixes: [string, PebbleIconName][]; names: Map<string, PebbleIconName> };
|
||||
let index: Index | null = null;
|
||||
|
||||
function buildIndex(): Index {
|
||||
const built: Index = { bundleIds: new Map(), prefixes: [], names: new Map() };
|
||||
for (const app of APP_ICONS) {
|
||||
for (const id of app.bundleIds) built.bundleIds.set(id.toLowerCase(), app.icon);
|
||||
for (const prefix of app.bundlePrefixes ?? []) built.prefixes.push([prefix.toLowerCase(), app.icon]);
|
||||
for (const name of app.names ?? []) built.names.set(name.toLowerCase(), app.icon);
|
||||
}
|
||||
return built;
|
||||
}
|
||||
|
||||
/**
|
||||
* The icon for a notification's source app, by bundle ID (ignoring case;
|
||||
* Apple treats bundle IDs case-insensitively), else by its display name;
|
||||
* null when the app isn't known.
|
||||
*/
|
||||
export function notificationAppIcon(bundleId: string, appName: string): PebbleIconName | null {
|
||||
index ??= buildIndex();
|
||||
const id = bundleId.trim().toLowerCase();
|
||||
if (id) {
|
||||
const byId = index.bundleIds.get(id) ?? index.prefixes.find(([prefix]) => id.startsWith(prefix))?.[1];
|
||||
if (byId) return byId;
|
||||
}
|
||||
const name = appName.trim().toLowerCase();
|
||||
return (name && index.names.get(name)) || null;
|
||||
}
|
||||
|
||||
/** The icon for an ANCS category (a CategoryID as a string); the bell for anything unknown. */
|
||||
export function notificationCategoryIcon(category: string): PebbleIconName {
|
||||
return CATEGORY_ICONS[Number(category)] ?? "generic";
|
||||
}
|
||||
|
||||
/** Every icon the table refers to, for tests. */
|
||||
export function allNotificationAppIcons(): PebbleIconName[] {
|
||||
return Array.from(new Set(APP_ICONS.map((app) => app.icon).concat(CATEGORY_ICONS)));
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
import { GrayImage } from '../graphics/image'
|
||||
import { renderIcon, type IconName } from '../graphics/icons'
|
||||
import type { GrayImage } from '../graphics/image'
|
||||
import { pebbleIcon } from '../graphics/pebble-icons'
|
||||
import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client'
|
||||
import { notificationAppIcon, notificationCategoryIcon } from './notification-app-icons'
|
||||
import { rememberNotificationSources } from './notification-sources'
|
||||
import type { AndroidNotification } from './notification-types'
|
||||
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
|
||||
export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons'
|
||||
export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons'
|
||||
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff
|
||||
let active: AncsClient | null = null
|
||||
@@ -32,36 +33,32 @@ export function onAndroidNotificationPosted(listener: (key: string) => void): ()
|
||||
// (they hear every change), and an open popup closes on the repaint that follows.
|
||||
export function onAndroidNotificationRemoved(_listener: (key: string) => void): () => void { return () => {} }
|
||||
export function onAndroidNotificationsChanged(listener: () => void): () => void { return onAndroidNotificationPosted(listener) }
|
||||
// ANCS does not provide app icons, so show one for the notification's ANCS
|
||||
// category; never fetch notification sources or content from a third-party
|
||||
// icon service. Indexed by CategoryID: Other, IncomingCall, MissedCall,
|
||||
// Voicemail, Social, Schedule, Email, News, HealthAndFitness,
|
||||
// BusinessAndFinance, Location, Entertainment.
|
||||
const CATEGORY_ICONS: readonly IconName[] = ['bell','phone-incoming','phone-missed','voicemail','message-circle',
|
||||
'calendar','mail','newspaper','heart-pulse','briefcase','map-pin','tv']
|
||||
function categoryIcon(category: string): GrayImage {
|
||||
return renderIcon(CATEGORY_ICONS[Number(category)] ?? 'bell',24)?.clone() ?? bell()
|
||||
}
|
||||
// Drawn fallback for when the SVG rasterizer fails.
|
||||
function bell(): GrayImage {
|
||||
const image = new GrayImage(24,24,0)
|
||||
image.fillRoundedRect(6,5,12,13,220,5)
|
||||
image.fillRect(4,17,16,2,220)
|
||||
image.fillRect(10,20,4,2,220)
|
||||
return image
|
||||
// ANCS does not provide app icons, so show a bundled Pebble icon for the
|
||||
// source app (by bundle ID or display name), else for the notification's
|
||||
// ANCS category. Never fetch notification sources or content from a
|
||||
// third-party icon service.
|
||||
function notificationIcon(notification: AndroidNotification): GrayImage {
|
||||
const name = notificationAppIcon(notification.packageName, notification.appName) ?? notificationCategoryIcon(notification.category)
|
||||
return pebbleIcon(name).clone()
|
||||
}
|
||||
export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) {
|
||||
// One icon per source, standing for its first notification.
|
||||
const sources = new Map<string, AndroidNotification>()
|
||||
for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n)
|
||||
const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons))
|
||||
return {icons: shown.map(n => categoryIcon(n.category)),keys: shown.map(n => n.key),stale:false}
|
||||
return {icons: shown.map(notificationIcon),keys: shown.map(n => n.key),stale:false}
|
||||
}
|
||||
export function readNotificationIconByKey(key: string, _allowStale: boolean) {
|
||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key)
|
||||
return {icon: notification ? categoryIcon(notification.category) : null,stale:false}
|
||||
return {icon: notification ? notificationIcon(notification) : null,stale:false}
|
||||
}
|
||||
// ANCS carries no attachments.
|
||||
// ANCS carries no attachments, avatars or custom layouts.
|
||||
export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
|
||||
return {picture: null,stale:false}
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
export function readNotificationLargeIcon(_notification: AndroidNotification, _size: number, _allowStale: boolean) {
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
export function readNotificationCustomView(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
|
||||
return {image: null,stale:false}
|
||||
}
|
||||
@@ -3,11 +3,12 @@ import { logCurrent, spanCurrent } from "./frame-timings";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { rememberNotificationSources } from "./notification-sources";
|
||||
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
|
||||
import { NOTIFICATION_ICON_SIZE } from "./notification-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
const ICON_SIZE = 24;
|
||||
const ICON_SIZE = NOTIFICATION_ICON_SIZE;
|
||||
/** Ask the native listener for every active source, including those below the list's display limit. */
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff;
|
||||
// Backstop TTL only: the icon caches are invalidated eagerly whenever a
|
||||
@@ -23,11 +24,6 @@ let cachedKeys: string[] = [];
|
||||
let cachedAtMs = 0;
|
||||
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
|
||||
const KEYED_ICON_CACHE_MAX = 128;
|
||||
/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */
|
||||
const pictureCache = new Map<string, {
|
||||
postTime: number; maxWidth: number; maxHeight: number; picture: GrayImage | null; atMs: number;
|
||||
}>();
|
||||
const PICTURE_CACHE_MAX = 8;
|
||||
let notificationListenerProxy: any = null;
|
||||
const notificationPostedListeners = new Set<(notificationKey: string) => void>();
|
||||
const notificationRemovedListeners = new Set<(notificationKey: string) => void>();
|
||||
@@ -159,48 +155,86 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not
|
||||
return { icon: icon?.clone() ?? null, stale: false };
|
||||
}
|
||||
|
||||
export type NotificationPictureResult = {
|
||||
export type NotificationImageResult = {
|
||||
/** Shared with the cache: blit it, don't draw on it. */
|
||||
picture: GrayImage | null;
|
||||
image: GrayImage | null;
|
||||
/** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */
|
||||
stale: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Images belonging to one notification (its picture, large icon or custom
|
||||
* layout), fetched to fit a box, under the allow-stale contract of
|
||||
* readNotificationIconByKey. Cached per post of the notification: an ongoing
|
||||
* notification updated in place (Dexcom redraws its graph and reading every
|
||||
* few minutes) fires no posted event, but its postTime moves. ICON_CACHE_MS
|
||||
* is the backstop here too. Under allowStale an earlier fetch stands in, if
|
||||
* it fits.
|
||||
*/
|
||||
class NotificationImageCache {
|
||||
private readonly entries = new Map<string, {
|
||||
postTime: number; maxWidth: number; maxHeight: number; image: GrayImage | null; atMs: number;
|
||||
}>();
|
||||
|
||||
constructor(private readonly maxEntries: number, private readonly span: string) {}
|
||||
|
||||
read(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
fetch: (width: number, height: number) => GrayImage | null,
|
||||
): NotificationImageResult {
|
||||
const width = Math.round(maxWidth);
|
||||
const height = Math.round(maxHeight);
|
||||
if (!global.isAndroid || width <= 0 || height <= 0) return { image: null, stale: false };
|
||||
|
||||
const now = Date.now();
|
||||
const cached = this.entries.get(notification.key);
|
||||
if (
|
||||
cached && cached.postTime === notification.postTime && cached.maxWidth === width
|
||||
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
|
||||
) {
|
||||
return { image: cached.image, stale: false };
|
||||
}
|
||||
if (allowStale) {
|
||||
const fits = cached?.image && cached.image.width <= width && cached.image.height <= height;
|
||||
return { image: fits ? cached!.image : null, stale: true };
|
||||
}
|
||||
|
||||
const image = spanCurrent(this.span, () => fetch(width, height));
|
||||
this.entries.delete(notification.key);
|
||||
this.entries.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, image, atMs: now });
|
||||
while (this.entries.size > this.maxEntries) {
|
||||
const oldestKey = this.entries.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
this.entries.delete(oldestKey);
|
||||
}
|
||||
return { image, stale: false };
|
||||
}
|
||||
}
|
||||
|
||||
/** Pictures and custom layouts are the size of a detail view, not icons: keep just the recently viewed few. */
|
||||
const pictureCache = new NotificationImageCache(8, "fetch-notification-picture");
|
||||
const customViewCache = new NotificationImageCache(8, "fetch-notification-custom-view");
|
||||
/** Large icons show on every list card drawn, so keep a screenful and then some. */
|
||||
const largeIconCache = new NotificationImageCache(32, "fetch-notification-large-icon");
|
||||
/** Avatars and logos: photographic tone, but undithered, which at icon size reads as noise. */
|
||||
const LARGE_ICON_TONE = { gamma: PHOTO_TONE.gamma, dither: false };
|
||||
|
||||
/**
|
||||
* The picture attached to a notification (hasPicture), scaled to fit
|
||||
* maxWidth x maxHeight with the photo tone and inverted if it is drawn on a
|
||||
* light background (GrayPacket.hasLightBackground), under the allow-stale contract of
|
||||
* readNotificationIconByKey. Cached per post of the notification: an ongoing
|
||||
* notification updated in place (Dexcom redraws its graph every few minutes)
|
||||
* fires no posted event, but its postTime moves. ICON_CACHE_MS is the
|
||||
* backstop here too. Under allowStale an earlier fetch stands in, if it fits.
|
||||
* light background (GrayPacket.hasLightBackground); see NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationPicture(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
): NotificationPictureResult {
|
||||
const width = Math.round(maxWidth);
|
||||
const height = Math.round(maxHeight);
|
||||
if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) {
|
||||
return { picture: null, stale: false };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const cached = pictureCache.get(notification.key);
|
||||
if (
|
||||
cached && cached.postTime === notification.postTime && cached.maxWidth === width
|
||||
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
|
||||
) {
|
||||
return { picture: cached.picture, stale: false };
|
||||
}
|
||||
if (allowStale) {
|
||||
const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height;
|
||||
return { picture: fits ? cached!.picture : null, stale: true };
|
||||
}
|
||||
|
||||
const picture = spanCurrent("fetch-notification-picture", () =>
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasPicture) return { image: null, stale: false };
|
||||
return pictureCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
|
||||
grayImageFromPacket(
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray(
|
||||
notification.key,
|
||||
@@ -212,16 +246,56 @@ export function readNotificationPicture(
|
||||
// lit panel on the glasses: show it light-on-dark.
|
||||
true,
|
||||
),
|
||||
),
|
||||
);
|
||||
pictureCache.delete(notification.key);
|
||||
pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now });
|
||||
while (pictureCache.size > PICTURE_CACHE_MAX) {
|
||||
const oldestKey = pictureCache.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
pictureCache.delete(oldestKey);
|
||||
}
|
||||
return { picture, stale: false };
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's large icon (hasLargeIcon: a sender's avatar, a site's
|
||||
* logo), scaled to fit size x size, a light background inverted as for
|
||||
* pictures; see NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationLargeIcon(
|
||||
notification: AndroidNotification,
|
||||
size: number,
|
||||
allowStale: boolean,
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasLargeIcon) return { image: null, stale: false };
|
||||
return largeIconCache.read(notification, size, size, allowStale, (width) =>
|
||||
grayImageFromPacket(
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationLargeIconGray(
|
||||
notification.key,
|
||||
width,
|
||||
LARGE_ICON_TONE.gamma,
|
||||
LARGE_ICON_TONE.dither,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's own layout (hasCustomView) drawn as the phone's shade
|
||||
* draws it, at most a pixel per dp and trimmed to what it draws, so usually
|
||||
* smaller than maxWidth x maxHeight; dark-on-light layouts come out
|
||||
* light-on-dark. See NotificationCustomViews.renderGray and
|
||||
* NotificationImageCache.
|
||||
*/
|
||||
export function readNotificationCustomView(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
): NotificationImageResult {
|
||||
if (!notification.hasCustomView) return { image: null, stale: false };
|
||||
return customViewCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) =>
|
||||
grayImageFromPacket(
|
||||
// Flat UI tone: it is mostly text and icons.
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationCustomViewGray(
|
||||
notification.key,
|
||||
width,
|
||||
height,
|
||||
1,
|
||||
false,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] {
|
||||
@@ -354,6 +428,9 @@ function normalizeNotification(value: any): AndroidNotification | null {
|
||||
actions,
|
||||
foregroundService: Boolean(value.foregroundService),
|
||||
hasPicture: Boolean(value.hasPicture),
|
||||
hasLargeIcon: Boolean(value.hasLargeIcon),
|
||||
hasCustomView: Boolean(value.hasCustomView),
|
||||
customText: Array.isArray(value.customText) ? value.customText.map((text: unknown) => String(text)).filter(Boolean) : [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
/** Shared notification model; Android names are retained for existing callers. */
|
||||
|
||||
/**
|
||||
* Side of a notification's (square) source icon wherever one is drawn: the
|
||||
* status bar, notification cards and views, the Glanceboard. 25 is the size
|
||||
* of the Pebble icons iPhone notifications use (app/graphics/pebble-icons.ts),
|
||||
* which are drawn on that pixel grid; Android's icons render at any size.
|
||||
*/
|
||||
export const NOTIFICATION_ICON_SIZE = 25;
|
||||
|
||||
export type AndroidNotificationAction = { index: number; title: string; enabled: boolean };
|
||||
export type AndroidNotification = {
|
||||
key: string; packageName: string; appName: string; title: string; text: string;
|
||||
@@ -9,6 +18,15 @@ export type AndroidNotification = {
|
||||
foregroundService?: boolean;
|
||||
/** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */
|
||||
hasPicture?: boolean;
|
||||
/** Carries a large icon (Android: a sender's avatar or logo), fetched separately by readNotificationLargeIcon. */
|
||||
hasLargeIcon?: boolean;
|
||||
/** Has its own layout (Android RemoteViews), drawn by readNotificationCustomView. */
|
||||
hasCustomView?: boolean;
|
||||
/**
|
||||
* The custom layout's text, one entry per text view, when the notification has no title or
|
||||
* text of its own (Dexcom's glucose reading and units).
|
||||
*/
|
||||
customText?: string[];
|
||||
/** Full message size reported by iOS, even when the fetched text is bounded. */
|
||||
messageSize?: number;
|
||||
};
|
||||
@@ -0,0 +1,159 @@
|
||||
import { fetchTextWithUserAgent, type TextResponse } from "../util/http";
|
||||
import { normalizeOpenAiCompatibleUrl } from "./openai-chat";
|
||||
|
||||
export type DiscoveredCompatibleModel = {
|
||||
/** Exact identifier to send as model in Chat Completions. */
|
||||
id: string;
|
||||
/** Human-friendly catalog name, or id when the server supplies no name. */
|
||||
name: string;
|
||||
/** true/false when advertised; null means the server did not say. */
|
||||
supportsTools: boolean | null;
|
||||
offline: boolean;
|
||||
deprecated: boolean;
|
||||
beta: boolean;
|
||||
};
|
||||
|
||||
type DiscoveryOptions = {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
/** Bypass the short-lived successful-response cache. */
|
||||
forceRefresh?: boolean;
|
||||
};
|
||||
|
||||
type DiscoveryResponse = Pick<TextResponse, "status" | "text">;
|
||||
type DiscoveryHttp = (url: string, init: {
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
timeoutMs: number;
|
||||
dontFollowRedirects: boolean;
|
||||
}) => Promise<DiscoveryResponse>;
|
||||
type DiscoveryDependencies = {
|
||||
http?: DiscoveryHttp;
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
type CacheEntry = {
|
||||
apiKey: string;
|
||||
fetchedAt: number;
|
||||
models: DiscoveredCompatibleModel[];
|
||||
};
|
||||
|
||||
export const COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS = 10_000;
|
||||
const CACHE_TTL_MS = 60_000;
|
||||
const MAX_MODELS = 500;
|
||||
const MAX_DISCOVERY_BODY_CHARS = 2_000_000;
|
||||
const cache = new Map<string, CacheEntry>();
|
||||
|
||||
/**
|
||||
* Discover models from the configured server's own /models endpoint. No URL
|
||||
* probing, inference calls, disk persistence, or automatic model selection.
|
||||
*/
|
||||
export async function discoverOpenAiCompatibleModels(
|
||||
options: DiscoveryOptions,
|
||||
dependencies: DiscoveryDependencies = {},
|
||||
): Promise<DiscoveredCompatibleModel[]> {
|
||||
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
|
||||
if (!baseUrl) throw new Error("Set a valid compatible API URL first");
|
||||
const apiKey = options.apiKey.trim();
|
||||
const now = dependencies.now ?? Date.now;
|
||||
const cached = cache.get(baseUrl);
|
||||
if (!options.forceRefresh && cached && cached.apiKey === apiKey && now() - cached.fetchedAt < CACHE_TTL_MS) {
|
||||
return cached.models;
|
||||
}
|
||||
|
||||
const http = dependencies.http ?? defaultHttp;
|
||||
const headers: Record<string, string> = {};
|
||||
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
||||
let response: DiscoveryResponse;
|
||||
try {
|
||||
response = await http(`${baseUrl}/models`, {
|
||||
method: "GET",
|
||||
headers,
|
||||
timeoutMs: COMPATIBLE_MODEL_DISCOVERY_TIMEOUT_MS,
|
||||
// Do not forward a credential to a redirect target.
|
||||
dontFollowRedirects: true,
|
||||
});
|
||||
} catch {
|
||||
throw new Error("Could not reach the configured server");
|
||||
}
|
||||
if (response.status < 200 || response.status >= 300) {
|
||||
throw new Error(describeHttpError(response.status));
|
||||
}
|
||||
|
||||
let body: any;
|
||||
let rawBody: string;
|
||||
try {
|
||||
rawBody = await response.text();
|
||||
if (rawBody.length > MAX_DISCOVERY_BODY_CHARS) throw new Error("too large");
|
||||
body = JSON.parse(rawBody);
|
||||
} catch {
|
||||
throw new Error("The server returned an invalid model list");
|
||||
}
|
||||
if (!body || !Array.isArray(body.data)) {
|
||||
throw new Error("The server did not return an OpenAI-compatible model list");
|
||||
}
|
||||
|
||||
const models = parseModels(body.data);
|
||||
cache.set(baseUrl, { apiKey, fetchedAt: now(), models });
|
||||
return models;
|
||||
}
|
||||
|
||||
/** Parse OpenAI data[] rows, retaining richer Venice metadata when present. */
|
||||
export function parseDiscoveredCompatibleModels(rows: any[]): DiscoveredCompatibleModel[] {
|
||||
return parseModels(rows);
|
||||
}
|
||||
|
||||
function parseModels(rows: any[]): DiscoveredCompatibleModel[] {
|
||||
const byId = new Map<string, DiscoveredCompatibleModel>();
|
||||
for (const row of rows) {
|
||||
if (!row || typeof row.id !== "string" || !row.id.trim()) continue;
|
||||
const id = row.id.trim();
|
||||
if (byId.has(id)) continue;
|
||||
const spec = row.model_spec && typeof row.model_spec === "object" ? row.model_spec : null;
|
||||
const capabilities = spec?.capabilities && typeof spec.capabilities === "object" ? spec.capabilities : null;
|
||||
const catalogName = typeof spec?.name === "string" ? spec.name.trim() : "";
|
||||
const genericName = typeof row.name === "string" ? row.name.trim() : "";
|
||||
byId.set(id, {
|
||||
id,
|
||||
name: catalogName || genericName || id,
|
||||
supportsTools: typeof capabilities?.supportsFunctionCalling === "boolean"
|
||||
? capabilities.supportsFunctionCalling
|
||||
: null,
|
||||
offline: spec?.offline === true,
|
||||
deprecated: Boolean(spec?.deprecation),
|
||||
beta: spec?.betaModel === true || spec?.beta === true,
|
||||
});
|
||||
if (byId.size >= MAX_MODELS) break;
|
||||
}
|
||||
return Array.from(byId.values()).sort((a, b) => {
|
||||
if (a.offline !== b.offline) return a.offline ? 1 : -1;
|
||||
const aTools = a.supportsTools === true ? 0 : a.supportsTools === null ? 1 : 2;
|
||||
const bTools = b.supportsTools === true ? 0 : b.supportsTools === null ? 1 : 2;
|
||||
if (aTools !== bTools) return aTools - bTools;
|
||||
return (a.name || a.id).localeCompare(b.name || b.id) || a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
|
||||
async function defaultHttp(url: string, init: {
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
timeoutMs: number;
|
||||
dontFollowRedirects: boolean;
|
||||
}): Promise<DiscoveryResponse> {
|
||||
return fetchTextWithUserAgent(url, init);
|
||||
}
|
||||
|
||||
function describeHttpError(status: number): string {
|
||||
if (status === 401) return "The server rejected the API key (HTTP 401)";
|
||||
if (status === 403) return "The API key cannot list these models (HTTP 403)";
|
||||
if (status === 404) return "Model discovery is not supported at this URL (HTTP 404)";
|
||||
if (status === 429) return "The server is rate limiting requests (HTTP 429)";
|
||||
if (status >= 500) return `The server is unavailable (HTTP ${status})`;
|
||||
if (status >= 300 && status < 400) return `The server redirected the request; use its direct API URL (HTTP ${status})`;
|
||||
return `Model discovery failed (HTTP ${status})`;
|
||||
}
|
||||
|
||||
/** Test hook; does not affect any saved setting. */
|
||||
export function clearCompatibleModelDiscoveryCache(): void {
|
||||
cache.clear();
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { openSseRequest, type SseListener } from "./sse";
|
||||
|
||||
/** True only for Venice's official API root, never for lookalike hosts. */
|
||||
export function isVeniceApiBaseUrl(input: string): boolean {
|
||||
return normalizeOpenAiCompatibleUrl(input) === "https://api.venice.ai/api/v1";
|
||||
}
|
||||
|
||||
/** Streaming client for OpenAI-compatible Chat Completions APIs. */
|
||||
export type OpenAiCompatibleStreamOptions = LlmStreamOptions & {
|
||||
/** Explicit API root, for example https://api.venice.ai/api/v1 or https://host/api. */
|
||||
baseUrl: string;
|
||||
/** Venice-only web search; never sent to other compatible providers. */
|
||||
webSearch?: boolean;
|
||||
};
|
||||
|
||||
type PendingToolCall = {
|
||||
id: string;
|
||||
name: string;
|
||||
inputJson: string;
|
||||
};
|
||||
|
||||
/** Remove an optional endpoint suffix, but never guess a provider's /v1 or /api prefix. */
|
||||
export function normalizeOpenAiCompatibleUrl(input: string): string {
|
||||
let url = input.trim().replace(/\/+$/, "");
|
||||
url = url.replace(/\/chat\/completions$/i, "").replace(/\/+$/, "");
|
||||
return /^https?:\/\/[^/\s?#@\\]+(?:\/[^\s?#\\]*)?$/i.test(url) ? url : "";
|
||||
}
|
||||
|
||||
export function streamOpenAiCompatibleChat(options: OpenAiCompatibleStreamOptions): LlmStreamHandle {
|
||||
const apiKey = options.apiKey.trim();
|
||||
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
|
||||
let request: { cancel(): void } | null = null;
|
||||
let settled = false;
|
||||
let text = "";
|
||||
let finishReason: string | null = null;
|
||||
const toolCalls = new Map<number, PendingToolCall>();
|
||||
const providerMetadata: Record<string, any> = {};
|
||||
const reasoningDetails = new ReasoningDetailsAccumulator();
|
||||
|
||||
const fail = (message: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
request?.cancel();
|
||||
options.onError(message);
|
||||
};
|
||||
|
||||
const done = () => {
|
||||
if (settled) return;
|
||||
const hasToolCalls = toolCalls.size > 0;
|
||||
// Never execute calls from a truncated stream or a length-limited response.
|
||||
if (hasToolCalls && finishReason !== "tool_calls" && finishReason !== "stop") {
|
||||
fail("OpenAI-compatible stream ended before its tool calls completed");
|
||||
return;
|
||||
}
|
||||
if (!text && !hasToolCalls) {
|
||||
fail("OpenAI-compatible provider ended the turn without a reply");
|
||||
return;
|
||||
}
|
||||
|
||||
const content: LlmContentBlock[] = [];
|
||||
if (text) content.push({ type: "text", text });
|
||||
const details = reasoningDetails.entries();
|
||||
if (details.length) providerMetadata.reasoning_details = details;
|
||||
if (Object.keys(providerMetadata).length) {
|
||||
content.push({ type: "provider_item", provider: "openai-compatible", item: providerMetadata });
|
||||
}
|
||||
for (const index of Array.from(toolCalls.keys()).sort((a, b) => a - b)) {
|
||||
const call = toolCalls.get(index)!;
|
||||
if (!call.name) {
|
||||
fail("OpenAI-compatible provider returned a tool call without a name");
|
||||
return;
|
||||
}
|
||||
let input: any = {};
|
||||
if (call.inputJson.trim()) {
|
||||
try {
|
||||
input = JSON.parse(call.inputJson);
|
||||
} catch {
|
||||
fail("OpenAI-compatible provider returned malformed tool arguments");
|
||||
return;
|
||||
}
|
||||
}
|
||||
content.push({
|
||||
type: "tool_use",
|
||||
id: call.id || `call_${index + 1}`,
|
||||
name: call.name,
|
||||
input,
|
||||
});
|
||||
}
|
||||
|
||||
settled = true;
|
||||
request?.cancel();
|
||||
options.onDone({
|
||||
text,
|
||||
content,
|
||||
stopReason: finishReason === "content_filter" ? "refusal"
|
||||
: hasToolCalls ? "tool_use"
|
||||
: finishReason === "length" ? "max_tokens" : "end_turn",
|
||||
});
|
||||
};
|
||||
|
||||
if (!baseUrl || !options.model.trim()) {
|
||||
setTimeout(() => fail(!baseUrl
|
||||
? "Set a valid OpenAI-compatible API URL"
|
||||
: "Set an OpenAI-compatible model ID"), 0);
|
||||
return { cancel: () => { settled = true; } };
|
||||
}
|
||||
|
||||
const body: any = {
|
||||
model: options.model.trim(),
|
||||
messages: buildChatMessages(options),
|
||||
stream: true,
|
||||
};
|
||||
// Let the provider choose its default unless a one-shot feature supplies a cap.
|
||||
if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens;
|
||||
if (options.tools?.length) {
|
||||
body.tools = options.tools.map((tool) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
},
|
||||
}));
|
||||
}
|
||||
// Provider-specific parameters must never leak to another endpoint, even
|
||||
// when a saved Venice preference is still enabled.
|
||||
if (options.webSearch === true && isVeniceApiBaseUrl(baseUrl)) {
|
||||
body.venice_parameters = { enable_web_search: "on" };
|
||||
}
|
||||
// Do not send provider-specific reasoning fields by default. Compatible
|
||||
// servers vary widely; advanced reasoning controls can be added separately.
|
||||
|
||||
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
||||
|
||||
const listener: SseListener = {
|
||||
onLine: (line: string) => {
|
||||
if (settled || !line.startsWith("data:")) return;
|
||||
const payload = line.slice(5).trim();
|
||||
if (!payload) return;
|
||||
if (payload === "[DONE]") {
|
||||
if (finishReason || (!toolCalls.size && text)) done();
|
||||
else fail("OpenAI-compatible stream ended before the response completed");
|
||||
return;
|
||||
}
|
||||
let event: any;
|
||||
try {
|
||||
event = JSON.parse(payload);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event?.error) {
|
||||
fail(`OpenAI-compatible: ${String(event.error.message ?? event.error)}`);
|
||||
return;
|
||||
}
|
||||
const choice = event?.choices?.[0];
|
||||
if (!choice) return; // Usage and provider-metadata chunks may have no choices.
|
||||
const delta = choice.delta ?? {};
|
||||
if (typeof delta.content === "string" && delta.content) {
|
||||
text += delta.content;
|
||||
options.onTextDelta?.(delta.content, text);
|
||||
}
|
||||
// Reasoning is not displayed. Preserve provider metadata for replay in
|
||||
// multi-turn tool loops when the service supplies it.
|
||||
if (Array.isArray(delta.reasoning_details)) reasoningDetails.add(delta.reasoning_details);
|
||||
if (delta.thought_signature !== undefined) providerMetadata.thought_signature = delta.thought_signature;
|
||||
if (Array.isArray(delta.tool_calls)) {
|
||||
delta.tool_calls.forEach((call: any, position: number) => {
|
||||
const index = Number.isInteger(call?.index) ? Number(call.index) : position;
|
||||
const pending = toolCalls.get(index) ?? { id: "", name: "", inputJson: "" };
|
||||
if (call?.id) pending.id = String(call.id);
|
||||
if (call?.function?.name) pending.name = String(call.function.name);
|
||||
if (typeof call?.function?.arguments === "string") pending.inputJson += call.function.arguments;
|
||||
toolCalls.set(index, pending);
|
||||
});
|
||||
}
|
||||
if (choice.finish_reason) finishReason = String(choice.finish_reason);
|
||||
},
|
||||
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
|
||||
onComplete: () => {
|
||||
// Some compatible servers close after the finish chunk without [DONE].
|
||||
if (finishReason) done();
|
||||
else fail("OpenAI-compatible stream ended before the response completed");
|
||||
},
|
||||
onFailure: (message: string) => fail(`OpenAI-compatible connection failed: ${String(message)}`),
|
||||
};
|
||||
|
||||
try {
|
||||
request = openSseRequest(`${baseUrl}/chat/completions`, JSON.stringify(body), headers, listener);
|
||||
} catch (error) {
|
||||
setTimeout(() => fail(`OpenAI-compatible request failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => { settled = true; } };
|
||||
}
|
||||
|
||||
return {
|
||||
cancel: () => {
|
||||
settled = true;
|
||||
try { request?.cancel(); } catch { /* ignore */ }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** String fields of a reasoning_details entry that stream in pieces. */
|
||||
const REASONING_DETAIL_TEXT_FIELDS = ["text", "summary", "data"];
|
||||
|
||||
/**
|
||||
* Rebuilds the complete reasoning_details array (OpenRouter's format) from
|
||||
* streamed deltas. Each delta carries fragments; fragments with the same
|
||||
* `index` belong to one entry, so their text, summary, and data concatenate
|
||||
* and their other fields (e.g. a signature that arrives after the text) fill
|
||||
* in. Entries without an index are kept as sent. The replayed array must
|
||||
* match what the model produced, so it is reassembled whole rather than
|
||||
* replaying only the latest chunk.
|
||||
*/
|
||||
class ReasoningDetailsAccumulator {
|
||||
private readonly ordered: Record<string, any>[] = [];
|
||||
private readonly byIndex = new Map<number, Record<string, any>>();
|
||||
|
||||
add(fragments: any[]): void {
|
||||
for (const fragment of fragments) {
|
||||
if (!fragment || typeof fragment !== "object" || Array.isArray(fragment)) continue;
|
||||
const index = Number.isInteger(fragment.index) ? Number(fragment.index) : null;
|
||||
const existing = index === null ? undefined : this.byIndex.get(index);
|
||||
if (!existing) {
|
||||
const entry = { ...fragment };
|
||||
this.ordered.push(entry);
|
||||
if (index !== null) this.byIndex.set(index, entry);
|
||||
continue;
|
||||
}
|
||||
for (const [key, value] of Object.entries(fragment)) {
|
||||
if (value === undefined || value === null) continue;
|
||||
if (REASONING_DETAIL_TEXT_FIELDS.includes(key) && typeof value === "string" && typeof existing[key] === "string") {
|
||||
existing[key] += value;
|
||||
} else if (existing[key] === undefined || existing[key] === null) {
|
||||
existing[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries(): Record<string, any>[] {
|
||||
return this.ordered;
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert neutral conversation history to Chat Completions messages. */
|
||||
function buildChatMessages(options: OpenAiCompatibleStreamOptions): any[] {
|
||||
const messages: any[] = [];
|
||||
if (options.system) messages.push({ role: "system", content: options.system });
|
||||
|
||||
for (const message of options.messages) {
|
||||
if (typeof message.content === "string") {
|
||||
messages.push({ role: message.role, content: message.content });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (message.role === "assistant") {
|
||||
let text = "";
|
||||
const toolCalls: any[] = [];
|
||||
let providerMetadata: Record<string, any> = {};
|
||||
for (const block of message.content) {
|
||||
if (block.type === "text") text += block.text;
|
||||
else if (block.type === "tool_use") {
|
||||
toolCalls.push({
|
||||
id: block.id,
|
||||
type: "function",
|
||||
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
|
||||
});
|
||||
} else if (block.type === "provider_item" && block.provider === "openai-compatible") {
|
||||
providerMetadata = { ...providerMetadata, ...block.item };
|
||||
}
|
||||
}
|
||||
if (text || toolCalls.length) {
|
||||
messages.push({
|
||||
role: "assistant",
|
||||
content: text || (toolCalls.length ? null : ""),
|
||||
...(toolCalls.length ? { tool_calls: toolCalls } : {}),
|
||||
...providerMetadata,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let pendingText = "";
|
||||
const flushText = () => {
|
||||
if (!pendingText) return;
|
||||
messages.push({ role: "user", content: pendingText });
|
||||
pendingText = "";
|
||||
};
|
||||
for (const block of message.content) {
|
||||
if (block.type === "text") pendingText += block.text;
|
||||
else if (block.type === "tool_result") {
|
||||
flushText();
|
||||
messages.push({
|
||||
role: "tool",
|
||||
tool_call_id: block.tool_use_id,
|
||||
content: block.is_error ? `Error: ${block.content}` : block.content,
|
||||
});
|
||||
}
|
||||
}
|
||||
flushText();
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
function describeHttpError(code: number, body: string): string {
|
||||
let detail = "";
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
detail = String(parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? "");
|
||||
} catch {
|
||||
// Non-JSON error body.
|
||||
}
|
||||
switch (code) {
|
||||
case 401: return "OpenAI-compatible: invalid API key";
|
||||
case 403: return "OpenAI-compatible: API key lacks permission";
|
||||
case 429: return "OpenAI-compatible: rate limited, try again shortly";
|
||||
case 500:
|
||||
case 502:
|
||||
case 503:
|
||||
case 504:
|
||||
return "OpenAI-compatible: service unavailable, try again shortly";
|
||||
default: return `OpenAI-compatible: HTTP ${code}${detail ? ` (${detail})` : ""}`;
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -201,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
function sessionConfig(language: string): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -209,7 +209,7 @@ function sessionConfig(): object {
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model: MODEL_ID },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
/** Platform-neutral process resource sample, shared by resource-usage.ts and resource-usage.ios.ts. */
|
||||
|
||||
export type ResourceSample = {
|
||||
/** Monotonic clock (counting deep sleep), ms. */
|
||||
atMs: number;
|
||||
/** Cumulative CPU time of every thread in the process, ms. */
|
||||
cpuMs: number;
|
||||
threads: number;
|
||||
/** One value per entry of the platform's MEMORY_SERIES, MiB. */
|
||||
memoryMb: number[];
|
||||
/** Platform-specific details for the latest sample (peaks, limits, thermal state). */
|
||||
status: string;
|
||||
};
|
||||
|
||||
export function kbToMb(kb: number): number {
|
||||
return Number.isFinite(kb) ? Math.max(0, kb) / 1024 : 0;
|
||||
}
|
||||
|
||||
export function formatMb(value: number): string {
|
||||
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
|
||||
return `${Math.round(value)}M`;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
|
||||
|
||||
export * from "./resource-usage-types";
|
||||
|
||||
declare const FaceclawResourceUsage: any;
|
||||
|
||||
/**
|
||||
* Memory series charted by the resource monitor, headline first: the physical
|
||||
* footprint (what jetsam enforces its limit against, and what Xcode's memory
|
||||
* gauge shows), resident memory, and bytes in use across the malloc zones.
|
||||
*/
|
||||
export const MEMORY_SERIES: readonly string[] = ["footprint", "resident", "malloc"];
|
||||
|
||||
const THERMAL_STATES = ["nominal", "fair", "serious", "critical"];
|
||||
|
||||
export function sampleResourceUsage(): ResourceSample {
|
||||
const raw = JSON.parse(String(FaceclawResourceUsage.sample())) as Record<string, number | boolean | undefined>;
|
||||
const kb = (key: string) => kbToMb(Number(raw[key] ?? 0));
|
||||
const footprintMb = kb("footprintKb");
|
||||
const peakMb = kb("footprintPeakKb");
|
||||
const availableMb = kb("availableKb");
|
||||
const thermal = THERMAL_STATES[Number(raw.thermalState)];
|
||||
return {
|
||||
atMs: Number(raw.atMs),
|
||||
cpuMs: Number(raw.cpuMs ?? 0),
|
||||
threads: Number(raw.threads ?? 0),
|
||||
memoryMb: [footprintMb, kb("residentKb"), kb("mallocKb")],
|
||||
status: [
|
||||
// Footprint plus the remaining headroom is the jetsam limit. No headroom
|
||||
// means no limit applies (the simulator) or it was just exceeded.
|
||||
availableMb > 0 ? `limit ${formatMb(footprintMb + availableMb)}` : "",
|
||||
peakMb > 0 ? `peak ${formatMb(peakMb)}` : "",
|
||||
thermal ? `thermal ${thermal}` : "",
|
||||
raw.lowPower === true ? "Low Power Mode" : "",
|
||||
].filter(Boolean).join(" · "),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { formatMb, kbToMb, type ResourceSample } from "./resource-usage-types";
|
||||
|
||||
export * from "./resource-usage-types";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Memory series charted by the resource monitor, headline first: RSS from
|
||||
* /proc/self/status (what Android records at exit), the native malloc heap,
|
||||
* and the used Java heap.
|
||||
*/
|
||||
export const MEMORY_SERIES: readonly string[] = ["RSS", "native", "Java"];
|
||||
|
||||
export function sampleResourceUsage(): ResourceSample {
|
||||
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
|
||||
const peakRssMb = kbToMb(Number(raw[7]));
|
||||
const javaCommittedMb = kbToMb(Number(raw[5]));
|
||||
return {
|
||||
atMs: Number(raw[0]),
|
||||
cpuMs: Number(raw[1]),
|
||||
memoryMb: [kbToMb(Number(raw[2])), kbToMb(Number(raw[3])), kbToMb(Number(raw[4]))],
|
||||
threads: Number(raw[6]),
|
||||
status: [
|
||||
peakRssMb > 0 ? `peak RSS ${formatMb(peakRssMb)}` : "",
|
||||
`Java heap ${formatMb(javaCommittedMb)} committed`,
|
||||
].filter(Boolean).join(" · "),
|
||||
};
|
||||
}
|
||||
@@ -50,6 +50,8 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
// A hint, not a restriction: other languages are still recognized.
|
||||
...(this.options.language ? { language_hints: [this.options.language] } : {}),
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
declare const FaceclawSpeechAnalyzer: any;
|
||||
|
||||
/**
|
||||
* How long finish() waits for the last results: the model only has to settle
|
||||
* the few seconds of audio it hadn't finalized yet.
|
||||
*/
|
||||
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
|
||||
|
||||
/**
|
||||
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
|
||||
* whose last word ends within this much before that (the model's word ends
|
||||
* against the detector's level threshold) leaves nothing before the pause
|
||||
* unfinalized, so its paragraph is complete.
|
||||
*/
|
||||
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
|
||||
|
||||
type AnalyzerEvent = {
|
||||
kind: string;
|
||||
message?: string;
|
||||
locale?: string;
|
||||
text?: string;
|
||||
final?: boolean;
|
||||
start?: number | null;
|
||||
end?: number | null;
|
||||
speechEnd?: number | null;
|
||||
};
|
||||
|
||||
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
|
||||
export function speechAnalyzerSupported(): boolean {
|
||||
try {
|
||||
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On-device long-form transcription for continuous capture, wrapped like a
|
||||
* cloud provider: it buffers audio until the model is ready (downloading it on
|
||||
* first use) and gets pause commits.
|
||||
*/
|
||||
export function createSpeechAnalyzerClient(
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): ReconnectingSttClient {
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
|
||||
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
|
||||
continuous,
|
||||
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
|
||||
* The model finalizes text in its own pieces, a few seconds behind, and can't
|
||||
* be made to settle at a pause; this gathers its pieces into paragraphs ended
|
||||
* by pauses, like the cloud providers' segments. A paragraph is shown as
|
||||
* provisional text and sent as one final (paragraphBreakAfter) once a final
|
||||
* reaches its pause, and the last one as a plain final when the audio ends.
|
||||
*
|
||||
* Only finals place breaks, by when their last word ends (the model's
|
||||
* word-end times are accurate; first words' starts reach back into the
|
||||
* silence before them, and volatile results only carry the range of all audio
|
||||
* not finalized yet).
|
||||
*/
|
||||
export class SpeechAnalyzerSttClient implements CloudSttClient {
|
||||
private native: any = null;
|
||||
private samples = 0;
|
||||
private finishing = false;
|
||||
// Audio times (s) of pauses whose paragraph hasn't ended yet.
|
||||
private pauses: number[] = [];
|
||||
// Finalized text since the last paragraph break, when its last word ends,
|
||||
// and the model's provisional rest.
|
||||
private paragraph: string[] = [];
|
||||
private paragraphEnd: number | null = null;
|
||||
private volatile = "";
|
||||
|
||||
constructor(private readonly options: CloudSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.native) return;
|
||||
const native = FaceclawSpeechAnalyzer.new();
|
||||
native.eventHandler = (json: string) => {
|
||||
if (this.native === native) this.receive(JSON.parse(json));
|
||||
};
|
||||
this.native = native;
|
||||
native.startWithLanguage(this.options.language ?? "");
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (!this.native || this.finishing || !pcm.length) return;
|
||||
this.samples += pcm.length >> 1;
|
||||
const copy = pcm.slice();
|
||||
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
|
||||
}
|
||||
|
||||
/** A pause: it ends the paragraph once the words before it are final. */
|
||||
commitSegment(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
|
||||
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.finishing = true;
|
||||
this.native.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
const native = this.native;
|
||||
this.native = null;
|
||||
native?.cancel();
|
||||
}
|
||||
|
||||
private receive(event: AnalyzerEvent): void {
|
||||
switch (event.kind) {
|
||||
case "status":
|
||||
this.options.onStatus(event.message ?? "");
|
||||
break;
|
||||
case "ready":
|
||||
this.options.onStatus("Listening (on-device)...");
|
||||
this.options.onReady?.();
|
||||
break;
|
||||
case "transcript":
|
||||
this.transcript(event);
|
||||
break;
|
||||
case "finished": {
|
||||
const text = this.liveText();
|
||||
this.paragraph = [];
|
||||
this.volatile = "";
|
||||
this.stop();
|
||||
this.options.onTranscript({ text, isFinal: true });
|
||||
break;
|
||||
}
|
||||
case "error":
|
||||
this.stop();
|
||||
this.options.onError(event.message ?? "On-device transcription failed.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private transcript(event: AnalyzerEvent): void {
|
||||
const text = (event.text ?? "").trim();
|
||||
let ended = false;
|
||||
if (!event.final) {
|
||||
this.volatile = text;
|
||||
} else {
|
||||
this.volatile = "";
|
||||
if (text) {
|
||||
const end = event.speechEnd ?? event.end ?? null;
|
||||
// Words ending after a pause come after it. (One whose words run
|
||||
// across it, if the model didn't split there, breaks a little early.)
|
||||
if (end !== null && this.pauses.some((pause) => end > pause)) {
|
||||
this.pauses = this.pauses.filter((pause) => pause >= end);
|
||||
this.endParagraph();
|
||||
}
|
||||
this.paragraph.push(text);
|
||||
this.paragraphEnd = end;
|
||||
ended = this.endCompleteParagraph();
|
||||
}
|
||||
}
|
||||
// The paragraph's final already cleared the provisional text.
|
||||
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
/** End the paragraph if its last word ends just before a pause. */
|
||||
private endCompleteParagraph(): boolean {
|
||||
const end = this.paragraphEnd;
|
||||
if (!this.paragraph.length || end === null) return false;
|
||||
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
|
||||
if (!this.pauses.some(reached)) return false;
|
||||
this.pauses = this.pauses.filter((pause) => !reached(pause));
|
||||
this.endParagraph();
|
||||
return true;
|
||||
}
|
||||
|
||||
private endParagraph(): void {
|
||||
const text = this.paragraph.join(" ");
|
||||
this.paragraph = [];
|
||||
this.paragraphEnd = null;
|
||||
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
|
||||
}
|
||||
|
||||
private liveText(): string {
|
||||
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
import { getStringSetting, setStringSetting } from "./settings-store";
|
||||
import { buildQuery, SpotifyApiError, SpotifyClient, type SpotifyFetch, type SpotifyTokenSource } from "./spotify-web-api";
|
||||
|
||||
/**
|
||||
* Spotify sign-in: OAuth authorization code with PKCE against the user's own
|
||||
* client ID (no client secret), redirected to a loopback address that the
|
||||
* phone catches (FaceclawWebSignIn on iOS). Spotify accepts http only for
|
||||
* loopback IP literals, and custom-scheme redirects fail with "insecure
|
||||
* redirect URI" for apps created since 2025, so the redirect is
|
||||
* http://127.0.0.1 on a fixed port the user registers in the dashboard.
|
||||
*
|
||||
* The tokens are stored with the client ID they were issued to; changing the
|
||||
* client ID setting signs out.
|
||||
*/
|
||||
|
||||
export const SPOTIFY_ACCOUNTS_BASE = "https://accounts.spotify.com";
|
||||
export const SPOTIFY_REDIRECT_PORT = 43823;
|
||||
export const SPOTIFY_REDIRECT_PATH = "/spotify-callback";
|
||||
export const SPOTIFY_REDIRECT_URI = `http://127.0.0.1:${SPOTIFY_REDIRECT_PORT}${SPOTIFY_REDIRECT_PATH}`;
|
||||
export const SPOTIFY_DASHBOARD_URL = "https://developer.spotify.com/dashboard";
|
||||
|
||||
export const SPOTIFY_CLIENT_ID_KEY = "music.spotify.clientId";
|
||||
export const SPOTIFY_ACCOUNT_KEY = "music.spotify.account";
|
||||
|
||||
export const SPOTIFY_SCOPES = [
|
||||
"user-read-playback-state",
|
||||
"user-modify-playback-state",
|
||||
"user-read-currently-playing",
|
||||
"user-read-recently-played",
|
||||
"playlist-read-private",
|
||||
"playlist-read-collaborative",
|
||||
"user-library-read",
|
||||
"user-library-modify",
|
||||
"user-follow-read",
|
||||
];
|
||||
|
||||
/** Refresh this long before Spotify's stated expiry (an hour), so a token never lapses mid-request. */
|
||||
const EXPIRY_MARGIN_MS = 60_000;
|
||||
|
||||
export type SpotifyAccount = {
|
||||
clientId: string;
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
/** Epoch ms. */
|
||||
expiresAtMs: number;
|
||||
userId: string;
|
||||
displayName: string;
|
||||
/** "premium", "free", ...; playback control needs premium. */
|
||||
product: string;
|
||||
};
|
||||
|
||||
export type SpotifyTokens = { accessToken: string; refreshToken: string; expiresAtMs: number };
|
||||
|
||||
/** The phone side of a sign-in: PKCE material and the browser sheet that catches the loopback redirect. */
|
||||
export type WebSignIn = {
|
||||
/** A fresh PKCE verifier and its S256 challenge. */
|
||||
createPkce(): { verifier: string; challenge: string };
|
||||
randomState(): string;
|
||||
/** Show the provider's page; resolves to the redirect URL, rejects with "cancelled" or why it failed. */
|
||||
open(authorizeUrl: string, port: number, path: string): Promise<string>;
|
||||
cancel(): void;
|
||||
};
|
||||
|
||||
export function spotifyClientId(): string {
|
||||
return getStringSetting(SPOTIFY_CLIENT_ID_KEY, "").trim();
|
||||
}
|
||||
|
||||
export function isValidSpotifyClientId(value: string): boolean {
|
||||
return /^[0-9a-f]{32}$/i.test(value.trim());
|
||||
}
|
||||
|
||||
/** The stored sign-in, or null when signed out or signed in with another client ID. */
|
||||
export function readSpotifyAccount(): SpotifyAccount | null {
|
||||
let raw: any;
|
||||
try {
|
||||
raw = JSON.parse(getStringSetting(SPOTIFY_ACCOUNT_KEY, "") || "null");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!raw || typeof raw.refreshToken !== "string" || !raw.refreshToken) return null;
|
||||
const clientId = spotifyClientId();
|
||||
if (!clientId || raw.clientId !== clientId) return null;
|
||||
return {
|
||||
clientId,
|
||||
accessToken: String(raw.accessToken ?? ""),
|
||||
refreshToken: raw.refreshToken,
|
||||
expiresAtMs: Number(raw.expiresAtMs) || 0,
|
||||
userId: String(raw.userId ?? ""),
|
||||
displayName: String(raw.displayName ?? ""),
|
||||
product: String(raw.product ?? ""),
|
||||
};
|
||||
}
|
||||
|
||||
export function saveSpotifyAccount(account: SpotifyAccount | null): void {
|
||||
setStringSetting(SPOTIFY_ACCOUNT_KEY, account ? JSON.stringify(account) : "");
|
||||
}
|
||||
|
||||
export function spotifyAuthorizeUrl(clientId: string, codeChallenge: string, state: string): string {
|
||||
return `${SPOTIFY_ACCOUNTS_BASE}/authorize${buildQuery({
|
||||
response_type: "code",
|
||||
client_id: clientId,
|
||||
scope: SPOTIFY_SCOPES.join(" "),
|
||||
redirect_uri: SPOTIFY_REDIRECT_URI,
|
||||
state,
|
||||
code_challenge_method: "S256",
|
||||
code_challenge: codeChallenge,
|
||||
})}`;
|
||||
}
|
||||
|
||||
/** The authorization code from Spotify's redirect; throws with a message for the glasses otherwise. */
|
||||
export function parseSpotifyRedirect(url: string, expectedState: string): string {
|
||||
const params = parseQueryString(url);
|
||||
if (params.error) {
|
||||
throw new Error(params.error === "access_denied" ? "Spotify sign-in was declined." : `Spotify sign-in failed: ${params.error}`);
|
||||
}
|
||||
if (params.state !== expectedState) throw new Error("Spotify sign-in failed: the reply didn't match this sign-in. Try again.");
|
||||
if (!params.code) throw new Error("Spotify sign-in failed: no authorization code in the reply.");
|
||||
return params.code;
|
||||
}
|
||||
|
||||
/** Query parameters of a URL (not URLSearchParams, which the NativeScript runtime mishandles). */
|
||||
export function parseQueryString(url: string): Record<string, string> {
|
||||
const query = url.split("#")[0]!.split("?").slice(1).join("?");
|
||||
const params: Record<string, string> = {};
|
||||
for (const part of query.split("&")) {
|
||||
if (!part) continue;
|
||||
const separator = part.indexOf("=");
|
||||
const key = separator < 0 ? part : part.slice(0, separator);
|
||||
const value = separator < 0 ? "" : part.slice(separator + 1);
|
||||
try {
|
||||
params[decodeURIComponent(key.replace(/\+/g, " "))] = decodeURIComponent(value.replace(/\+/g, " "));
|
||||
} catch {
|
||||
// Malformed escapes: skip the parameter.
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/** POST to Spotify's token endpoint (authorization_code or refresh_token grant). */
|
||||
export async function requestSpotifyTokens(
|
||||
fetchImpl: SpotifyFetch,
|
||||
params: Record<string, string>,
|
||||
nowMs: number,
|
||||
): Promise<SpotifyTokens> {
|
||||
let status = 0;
|
||||
let body = "";
|
||||
try {
|
||||
const response = await fetchImpl(`${SPOTIFY_ACCOUNTS_BASE}/api/token`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json" },
|
||||
body: buildQuery(params).slice(1),
|
||||
});
|
||||
status = response.status;
|
||||
body = await response.text();
|
||||
} catch (error) {
|
||||
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
|
||||
}
|
||||
let parsed: any = null;
|
||||
try {
|
||||
parsed = JSON.parse(body);
|
||||
} catch {
|
||||
// Reported below.
|
||||
}
|
||||
if (status >= 200 && status < 300 && typeof parsed?.access_token === "string") {
|
||||
const expiresIn = Number(parsed.expires_in);
|
||||
return {
|
||||
accessToken: parsed.access_token,
|
||||
// Spotify may rotate the refresh token; "" keeps the current one.
|
||||
refreshToken: typeof parsed.refresh_token === "string" ? parsed.refresh_token : "",
|
||||
expiresAtMs: nowMs + (Number.isFinite(expiresIn) && expiresIn > 0 ? expiresIn : 3600) * 1000,
|
||||
};
|
||||
}
|
||||
const error = typeof parsed?.error === "string" ? parsed.error : "";
|
||||
const description = typeof parsed?.error_description === "string" ? parsed.error_description : "";
|
||||
if (error === "invalid_grant" && params.grant_type === "refresh_token") {
|
||||
throw new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, error);
|
||||
}
|
||||
if (error === "invalid_client") {
|
||||
throw new SpotifyApiError("Spotify doesn't recognize the client ID. Check it under Spotify in the Music app menu.", status, error);
|
||||
}
|
||||
throw new SpotifyApiError(`Spotify sign-in failed: ${description || error || `HTTP ${status}`}`, status, error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Access tokens for the stored account: the current one until it nears
|
||||
* expiry, then one refresh shared by every caller waiting on it. A refresh
|
||||
* Spotify rejects (revoked access) signs out.
|
||||
*/
|
||||
export class SpotifySession implements SpotifyTokenSource {
|
||||
private refreshing: Promise<string> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly fetchImpl: SpotifyFetch,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
accessToken(forceRefresh: boolean): Promise<string> {
|
||||
const account = readSpotifyAccount();
|
||||
if (!account) return Promise.reject(new SpotifyApiError("Not signed in to Spotify.", 401, "signed_out"));
|
||||
if (!forceRefresh && account.accessToken && account.expiresAtMs - EXPIRY_MARGIN_MS > this.now()) {
|
||||
return Promise.resolve(account.accessToken);
|
||||
}
|
||||
this.refreshing ??= this.refresh(account).finally(() => {
|
||||
this.refreshing = null;
|
||||
});
|
||||
return this.refreshing;
|
||||
}
|
||||
|
||||
private async refresh(account: SpotifyAccount): Promise<string> {
|
||||
let tokens: SpotifyTokens;
|
||||
try {
|
||||
tokens = await requestSpotifyTokens(this.fetchImpl, {
|
||||
grant_type: "refresh_token",
|
||||
refresh_token: account.refreshToken,
|
||||
client_id: account.clientId,
|
||||
}, this.now());
|
||||
} catch (error) {
|
||||
if (error instanceof SpotifyApiError && error.reason === "invalid_grant" && readSpotifyAccount()?.refreshToken === account.refreshToken) {
|
||||
saveSpotifyAccount(null);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// A sign-out or another sign-in while this was in flight wins.
|
||||
if (readSpotifyAccount()?.refreshToken !== account.refreshToken) {
|
||||
throw new SpotifyApiError("Spotify sign-in changed; try again.", 401, "signed_out");
|
||||
}
|
||||
saveSpotifyAccount({
|
||||
...account,
|
||||
accessToken: tokens.accessToken,
|
||||
refreshToken: tokens.refreshToken || account.refreshToken,
|
||||
expiresAtMs: tokens.expiresAtMs,
|
||||
});
|
||||
return tokens.accessToken;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The whole sign-in: the browser sheet, the code exchange and the profile
|
||||
* lookup; stores and returns the account. Rejects with "cancelled" when the
|
||||
* user closes the sheet.
|
||||
*/
|
||||
export async function signInToSpotify(
|
||||
fetchImpl: SpotifyFetch,
|
||||
webSignIn: WebSignIn,
|
||||
now: () => number = Date.now,
|
||||
): Promise<SpotifyAccount> {
|
||||
const clientId = spotifyClientId();
|
||||
if (!isValidSpotifyClientId(clientId)) throw new Error("Enter your Spotify client ID first.");
|
||||
const { verifier, challenge } = webSignIn.createPkce();
|
||||
const state = webSignIn.randomState();
|
||||
const redirect = await webSignIn.open(spotifyAuthorizeUrl(clientId, challenge, state), SPOTIFY_REDIRECT_PORT, SPOTIFY_REDIRECT_PATH);
|
||||
const code = parseSpotifyRedirect(redirect, state);
|
||||
const tokens = await requestSpotifyTokens(fetchImpl, {
|
||||
grant_type: "authorization_code",
|
||||
code,
|
||||
redirect_uri: SPOTIFY_REDIRECT_URI,
|
||||
client_id: clientId,
|
||||
code_verifier: verifier,
|
||||
}, now());
|
||||
const user = await new SpotifyClient(fetchImpl, { accessToken: async () => tokens.accessToken }).me();
|
||||
const account: SpotifyAccount = {
|
||||
clientId,
|
||||
accessToken: tokens.accessToken,
|
||||
refreshToken: tokens.refreshToken,
|
||||
expiresAtMs: tokens.expiresAtMs,
|
||||
userId: user.id,
|
||||
displayName: user.displayName,
|
||||
product: user.product,
|
||||
};
|
||||
if (!account.refreshToken) throw new Error("Spotify sign-in failed: no refresh token in the reply.");
|
||||
saveSpotifyAccount(account);
|
||||
return account;
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import type { MediaBrowseItem, MediaBrowserApp, MediaBrowserBridge } from "./media-types";
|
||||
import { SPOTIFY_APP_NAME, type SpotifyPlayer } from "./spotify-player";
|
||||
import type { SpotifyClient, SpotifyItem, SpotifyPlayRequest } from "./spotify-web-api";
|
||||
|
||||
const ROOT_ROWS: MediaBrowseItem[] = [
|
||||
{ mediaId: "recent", title: "Recently played", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "playlists", title: "Playlists", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "liked", title: "Liked Songs", subtitle: "", browsable: true, playable: true },
|
||||
{ mediaId: "albums", title: "Albums", subtitle: "", browsable: true, playable: false },
|
||||
{ mediaId: "artists", title: "Artists", subtitle: "", browsable: true, playable: false },
|
||||
];
|
||||
|
||||
/** Separates a track's URI from the context it plays in ("" for none, "liked" for Liked Songs). */
|
||||
const TRACK_CONTEXT_SEPARATOR = "|";
|
||||
|
||||
export type SpotifyLibraryDeps = {
|
||||
client: SpotifyClient;
|
||||
/** Spotify's app id on this platform, which names the library in the browse picker. */
|
||||
packageName: string;
|
||||
player: SpotifyPlayer;
|
||||
isSignedIn(): boolean;
|
||||
/** The signed-in user's id (Liked Songs' context URI names it). */
|
||||
userId(): string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The signed-in user's Spotify library as a browse tree (Web API):
|
||||
*
|
||||
* root -> recent | playlists | liked | albums | artists
|
||||
* playlist:<id>, album:<id> -> tracks (browsable and playable)
|
||||
* artist:<id> -> albums
|
||||
* track:<uri>|<context> -> plays the track within its context
|
||||
*
|
||||
* Only playlists the user owns or collaborates on can list their tracks
|
||||
* (development-mode client IDs, since February 2026); others are play-only.
|
||||
*/
|
||||
export class SpotifyLibrary implements MediaBrowserBridge {
|
||||
private connected = false;
|
||||
/** Bumped on every connect/disconnect; answers to older requests are dropped. */
|
||||
private generation = 0;
|
||||
|
||||
/** This library's row in the browse picker. */
|
||||
readonly app: MediaBrowserApp;
|
||||
|
||||
constructor(private readonly deps: SpotifyLibraryDeps) {
|
||||
this.app = { packageName: deps.packageName, serviceClass: "", appName: SPOTIFY_APP_NAME };
|
||||
}
|
||||
|
||||
listBrowsableApps(): MediaBrowserApp[] {
|
||||
return this.deps.isSignedIn() ? [this.app] : [];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
if (app.packageName !== this.app.packageName) return Promise.reject(new Error("Not a Spotify library"));
|
||||
if (!this.deps.isSignedIn()) return Promise.reject(new Error("Sign in under Spotify in the Music app menu first."));
|
||||
this.connected = true;
|
||||
return Promise.resolve("root");
|
||||
}
|
||||
|
||||
async browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
if (!this.connected) throw new Error("Not connected");
|
||||
const generation = this.generation;
|
||||
const rows = await this.list(parentId);
|
||||
if (generation !== this.generation) throw new Error("Disconnected");
|
||||
return rows;
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
if (!this.connected) return;
|
||||
const request = this.playRequest(mediaId);
|
||||
if (request) void this.deps.player.startPlayback(request);
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.connected = false;
|
||||
this.generation++;
|
||||
}
|
||||
|
||||
/** What playing a browse row starts; null for rows that don't play. */
|
||||
playRequest(mediaId: string): SpotifyPlayRequest | null {
|
||||
if (mediaId === "liked") return { contextUri: this.likedContextUri() };
|
||||
const [kind, ...rest] = mediaId.split(":");
|
||||
const id = rest.join(":");
|
||||
switch (kind) {
|
||||
case "playlist":
|
||||
case "album":
|
||||
case "artist":
|
||||
return id ? { contextUri: `spotify:${kind}:${id}` } : null;
|
||||
case "track": {
|
||||
const separator = id.lastIndexOf(TRACK_CONTEXT_SEPARATOR);
|
||||
const uri = separator < 0 ? id : id.slice(0, separator);
|
||||
const context = separator < 0 ? "" : id.slice(separator + 1);
|
||||
if (!uri) return null;
|
||||
if (context === "liked") return { contextUri: this.likedContextUri(), offsetUri: uri };
|
||||
return context ? { contextUri: context, offsetUri: uri } : { uris: [uri] };
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async list(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
const client = this.deps.client;
|
||||
if (parentId === "root") return ROOT_ROWS;
|
||||
if (parentId === "recent") {
|
||||
// The history repeats tracks; keep each one's latest play.
|
||||
const seen = new Set<string>();
|
||||
return (await client.recentlyPlayed())
|
||||
.filter((played) => !seen.has(played.item.uri) && seen.add(played.item.uri))
|
||||
.map((played) => trackRow(played.item, played.contextUri));
|
||||
}
|
||||
if (parentId === "playlists") {
|
||||
const userId = this.deps.userId();
|
||||
return (await client.playlists()).map((playlist) => {
|
||||
const listable = playlist.collaborative || playlist.ownerId === userId;
|
||||
return {
|
||||
mediaId: `playlist:${playlist.id}`,
|
||||
title: playlist.name,
|
||||
subtitle: listable ? "" : playlist.ownerName,
|
||||
browsable: listable,
|
||||
playable: true,
|
||||
};
|
||||
});
|
||||
}
|
||||
if (parentId === "liked") return (await client.savedTracks()).map((item) => trackRow(item, "liked"));
|
||||
if (parentId === "albums") return (await client.savedAlbums()).map(albumRow);
|
||||
if (parentId === "artists") {
|
||||
return (await client.followedArtists()).map((artist) => ({
|
||||
mediaId: `artist:${artist.id}`,
|
||||
title: artist.name,
|
||||
subtitle: "",
|
||||
browsable: true,
|
||||
playable: true,
|
||||
}));
|
||||
}
|
||||
const [kind, ...rest] = parentId.split(":");
|
||||
const id = rest.join(":");
|
||||
if (kind === "playlist") return (await client.playlistItems(id)).map((item) => trackRow(item, `spotify:playlist:${id}`));
|
||||
if (kind === "album") return (await client.albumTracks(id)).map((item) => trackRow(item, `spotify:album:${id}`));
|
||||
if (kind === "artist") return (await client.artistAlbums(id)).map(albumRow);
|
||||
throw new Error(`Unknown Spotify library item: ${parentId}`);
|
||||
}
|
||||
|
||||
private likedContextUri(): string {
|
||||
return `spotify:user:${this.deps.userId()}:collection`;
|
||||
}
|
||||
}
|
||||
|
||||
function trackRow(item: SpotifyItem, contextUri: string): MediaBrowseItem {
|
||||
return {
|
||||
mediaId: `track:${item.uri}${contextUri ? `${TRACK_CONTEXT_SEPARATOR}${contextUri}` : ""}`,
|
||||
title: item.name,
|
||||
subtitle: item.artists,
|
||||
browsable: false,
|
||||
playable: true,
|
||||
};
|
||||
}
|
||||
|
||||
function albumRow(album: { id: string; name: string; artists: string }): MediaBrowseItem {
|
||||
return { mediaId: `album:${album.id}`, title: album.name, subtitle: album.artists, browsable: true, playable: true };
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
import {
|
||||
DEFAULT_MEDIA_STATE,
|
||||
interpolateMediaState,
|
||||
type MediaControllerBridge,
|
||||
type MediaControllerState,
|
||||
type MediaCustomAction,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
import {
|
||||
SpotifyApiError,
|
||||
type SpotifyClient,
|
||||
type SpotifyDevice,
|
||||
type SpotifyItem,
|
||||
type SpotifyPlayback,
|
||||
type SpotifyPlayRequest,
|
||||
type SpotifyRepeat,
|
||||
} from "./spotify-web-api";
|
||||
|
||||
export const SPOTIFY_APP_NAME = "Spotify";
|
||||
/** Spotify's iOS bundle id: the player's packageName in the media state on iOS. */
|
||||
export const SPOTIFY_IOS_BUNDLE_ID = "com.spotify.client";
|
||||
/** Spotify's Android package: its media session's owner, which the Web API player reports too. */
|
||||
export const SPOTIFY_ANDROID_PACKAGE = "com.spotify.music";
|
||||
|
||||
/**
|
||||
* Poll intervals while something listens. The Web API has no push channel, so
|
||||
* a track change shows up within one interval (sooner at a track's end, which
|
||||
* is polled for directly); requests count against the client ID's quota.
|
||||
*/
|
||||
const POLL_PLAYING_MS = 5_000;
|
||||
const POLL_PAUSED_MS = 10_000;
|
||||
const POLL_IDLE_MS = 20_000;
|
||||
const POLL_ERROR_MS = 30_000;
|
||||
/** Re-read soon after a command: Spotify takes a moment to report its effect. */
|
||||
const SETTLE_POLL_MS = 600;
|
||||
const SETTLE_POLLS = 2;
|
||||
const NOTICE_MS = 8_000;
|
||||
/** Re-read the queue this often while the Music app shows it, besides on every track change. */
|
||||
const QUEUE_MAX_AGE_MS = 30_000;
|
||||
/** At most this often, even on track changes (and after a failed read). */
|
||||
const QUEUE_MIN_INTERVAL_MS = 5_000;
|
||||
/** Album art: the smallest of Spotify's sizes (640/300/64) at least this wide. */
|
||||
const ART_MIN_WIDTH = 128;
|
||||
|
||||
const REPEAT_NEXT: Record<SpotifyRepeat, SpotifyRepeat> = { off: "context", context: "track", track: "off" };
|
||||
const REPEAT_LABEL: Record<SpotifyRepeat, string> = { off: "Repeat: off", context: "Repeat: all", track: "Repeat: one" };
|
||||
|
||||
export type SpotifyPlayerDeps = {
|
||||
client: SpotifyClient;
|
||||
/** The state's packageName: Spotify's app id on this platform. */
|
||||
packageName: string;
|
||||
isSignedIn(): boolean;
|
||||
/** Fetch an album-art image's encoded bytes; null on failure. */
|
||||
download(url: string): Promise<Uint8Array | null>;
|
||||
/** Encoded image bytes to the shared photo tone, fitted within maxSize; null when undecodable. */
|
||||
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
|
||||
/** The phone's output volume, for playback on this phone (Spotify won't set a phone's volume). */
|
||||
systemVolume: { get(): number; set(percent: number): void };
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Spotify Connect playback (on this phone or any of the user's devices)
|
||||
* through the Web API: state from polling GET /me/player while something
|
||||
* listens, commands as API calls whose failures (no device, no Premium)
|
||||
* become a short notice in the state.
|
||||
*/
|
||||
export class SpotifyPlayer implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Sign in to Spotify from the Music app menu to show and control Spotify.";
|
||||
readonly accessLabel = "Spotify sign-in needed";
|
||||
readonly accessAction = "open the menu";
|
||||
/** Jumping skips forward through the queue, so the chosen track ends up first. */
|
||||
readonly skipRebuildsQueue = true;
|
||||
/** When Spotify was last seen playing (epoch ms); 0 = not this run. */
|
||||
lastPlayingAtMs = 0;
|
||||
|
||||
private readonly now: () => number;
|
||||
private readonly listeners = new Set<(state: MediaControllerState) => void>();
|
||||
private playback: SpotifyPlayback | null = null;
|
||||
private fetchedAtMs = 0;
|
||||
private pollTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private polling: Promise<void> | null = null;
|
||||
private pollAgain = false;
|
||||
private settlePolls = 0;
|
||||
private lastSignature = "";
|
||||
private notice = "";
|
||||
private noticeUntilMs = 0;
|
||||
private noticeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private upcoming: SpotifyItem[] = [];
|
||||
/** The item the queue was read with; "" = never read. */
|
||||
private queueCurrentUri = "";
|
||||
/** When the queue was last read, or a read failed. */
|
||||
private queueFetchedAtMs = 0;
|
||||
private queueLoading = false;
|
||||
private art: { url: string; bytes: Uint8Array | null; decoded: Map<number, GrayImage | null> } | null = null;
|
||||
private artVersion = 0;
|
||||
private liked: { uri: string; value: boolean | null } | null = null;
|
||||
private volumeTarget: number | null = null;
|
||||
private volumeSending = false;
|
||||
|
||||
constructor(private readonly deps: SpotifyPlayerDeps) {
|
||||
this.now = deps.now ?? Date.now;
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
listener(this.snapshot());
|
||||
if (this.listeners.size === 1) this.schedulePoll(0);
|
||||
return () => {
|
||||
this.listeners.delete(listener);
|
||||
if (!this.listeners.size && !this.settlePolls) this.clearPollTimer();
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const notice = this.currentNotice();
|
||||
if (!this.deps.isSignedIn()) {
|
||||
return { ...DEFAULT_MEDIA_STATE, status: "Not signed in to Spotify.", notice };
|
||||
}
|
||||
const playback = this.playback;
|
||||
if (!playback) {
|
||||
return {
|
||||
...DEFAULT_MEDIA_STATE,
|
||||
accessEnabled: true,
|
||||
packageName: this.deps.packageName,
|
||||
appName: SPOTIFY_APP_NAME,
|
||||
status: "Nothing is playing on Spotify.",
|
||||
notice,
|
||||
};
|
||||
}
|
||||
const item = playback.item;
|
||||
const restricted = playback.device?.restricted === true;
|
||||
const allowed = (action: string) => !restricted && !playback.disallows.includes(action);
|
||||
const state: MediaControllerState = {
|
||||
available: true,
|
||||
accessEnabled: true,
|
||||
packageName: this.deps.packageName,
|
||||
appName: spotifyAppName(playback.device),
|
||||
playbackState: playback.playing ? "playing" : "paused",
|
||||
title: item?.name ?? (playback.playingType === "ad" ? "Advertisement" : ""),
|
||||
artist: item?.artists ?? "",
|
||||
album: item?.album ?? "",
|
||||
positionMs: item ? playback.progressMs : -1,
|
||||
durationMs: item?.durationMs ?? -1,
|
||||
playbackSpeed: playback.playing ? 1 : 0,
|
||||
status: "",
|
||||
canPlayPause: allowed(playback.playing ? "pausing" : "resuming"),
|
||||
canSkipNext: allowed("skipping_next"),
|
||||
canSkipPrevious: allowed("skipping_prev"),
|
||||
artVersion: this.artVersion,
|
||||
customActions: restricted ? [] : this.customActions(playback),
|
||||
notice,
|
||||
};
|
||||
return interpolateMediaState(state, this.fetchedAtMs, this.now());
|
||||
}
|
||||
|
||||
async start(): Promise<void> {}
|
||||
|
||||
async stop(): Promise<void> {}
|
||||
|
||||
async playPause(): Promise<void> {
|
||||
const playback = this.playback;
|
||||
if (!playback?.playing) {
|
||||
await this.startPlayback({});
|
||||
return;
|
||||
}
|
||||
await this.command(() => this.deps.client.pause(), () => {
|
||||
playback.progressMs = this.snapshot().positionMs;
|
||||
playback.playing = false;
|
||||
this.fetchedAtMs = this.now();
|
||||
});
|
||||
}
|
||||
|
||||
async skipNext(): Promise<void> {
|
||||
await this.command(() => this.deps.client.next());
|
||||
}
|
||||
|
||||
async skipPrevious(): Promise<void> {
|
||||
await this.command(() => this.deps.client.previous());
|
||||
}
|
||||
|
||||
/** Signed in; whether anything is playing is in the state. */
|
||||
hasAccess(): boolean {
|
||||
return this.deps.isSignedIn();
|
||||
}
|
||||
|
||||
/** Sign-in happens from the Music app menu, which shows its progress. */
|
||||
async requestAccess(): Promise<boolean> {
|
||||
return this.hasAccess();
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue ids are positions in getQueue (0 = the current item). The Web API
|
||||
* has no jump, so this skips forward that many times.
|
||||
*/
|
||||
async skipToQueueItem(id: number): Promise<void> {
|
||||
const steps = Math.round(id);
|
||||
if (steps <= 0 || steps > this.upcoming.length) return;
|
||||
await this.command(async () => {
|
||||
for (let step = 0; step < steps; step++) await this.deps.client.next();
|
||||
}, () => {
|
||||
const target = this.upcoming[steps - 1];
|
||||
if (this.playback && target) {
|
||||
this.playback.item = target;
|
||||
this.playback.progressMs = 0;
|
||||
this.fetchedAtMs = this.now();
|
||||
this.upcoming = this.upcoming.slice(steps);
|
||||
this.queueCurrentUri = target.uri;
|
||||
this.itemChanged(target);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
getMediaVolumePercent(): number {
|
||||
const device = this.playback?.device ?? null;
|
||||
if (!usesDeviceVolume(device)) return this.deps.systemVolume.get();
|
||||
return this.volumeTarget ?? device!.volumePercent;
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
const device = this.playback?.device ?? null;
|
||||
const volume = Math.max(0, Math.min(100, Math.round(volumePercent)));
|
||||
if (!usesDeviceVolume(device)) {
|
||||
this.deps.systemVolume.set(volume);
|
||||
return;
|
||||
}
|
||||
// Scrolling sends a value per step: keep one request in flight and send the latest after it.
|
||||
device!.volumePercent = volume;
|
||||
this.volumeTarget = volume;
|
||||
if (!this.volumeSending) void this.sendVolume(device!.id);
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
const art = this.art;
|
||||
if (!art?.bytes) return null;
|
||||
const size = Math.round(maxSize);
|
||||
if (!art.decoded.has(size)) art.decoded.set(size, this.deps.decodeArt(art.bytes, size));
|
||||
return art.decoded.get(size) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The current item, then what Spotify will play next. Read on demand (the
|
||||
* Music app asks while it paints), so nothing else pays for the request; a
|
||||
* stale list stays up, minus the items already played, until the new one
|
||||
* arrives.
|
||||
*/
|
||||
getQueue(): MediaQueueItem[] {
|
||||
const current = this.playback?.item;
|
||||
if (!current) return [];
|
||||
const age = this.now() - this.queueFetchedAtMs;
|
||||
if (!this.queueLoading && (age > QUEUE_MAX_AGE_MS || (this.queueCurrentUri !== current.uri && age > QUEUE_MIN_INTERVAL_MS))) {
|
||||
void this.loadQueue();
|
||||
}
|
||||
if (!this.queueCurrentUri) return [];
|
||||
if (this.queueCurrentUri !== current.uri) {
|
||||
const index = this.upcoming.findIndex((item) => item.uri === current.uri);
|
||||
if (index >= 0) {
|
||||
this.upcoming = this.upcoming.slice(index + 1);
|
||||
this.queueCurrentUri = current.uri;
|
||||
}
|
||||
}
|
||||
return [current, ...this.upcoming].map((item, index) => ({
|
||||
id: index,
|
||||
title: item.name,
|
||||
subtitle: item.artists,
|
||||
active: index === 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async sendCustomAction(action: string): Promise<void> {
|
||||
const playback = this.playback;
|
||||
if (!playback) return;
|
||||
const client = this.deps.client;
|
||||
if (action === "shuffle") {
|
||||
const shuffle = !playback.shuffle;
|
||||
await this.command(() => client.setShuffle(shuffle), () => {
|
||||
playback.shuffle = shuffle;
|
||||
// Shuffling reorders what comes next.
|
||||
this.queueFetchedAtMs = 0;
|
||||
});
|
||||
} else if (action === "repeat") {
|
||||
const repeat = REPEAT_NEXT[playback.repeat];
|
||||
await this.command(() => client.setRepeat(repeat), () => {
|
||||
playback.repeat = repeat;
|
||||
});
|
||||
} else if (action === "like") {
|
||||
const liked = this.liked;
|
||||
if (!liked || liked.value === null) return;
|
||||
const value = !liked.value;
|
||||
await this.command(() => (value ? client.saveToLibrary([liked.uri]) : client.removeFromLibrary([liked.uri])), () => {
|
||||
if (this.liked === liked) liked.value = value;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async refresh(): Promise<void> {
|
||||
await this.poll();
|
||||
}
|
||||
|
||||
/**
|
||||
* Play something (a browsed item, a search result) or, with an empty
|
||||
* request, resume. With no active device, starts on one the account has
|
||||
* open (preferring a phone). Resolves to whether it worked; a failure is
|
||||
* left as the state's notice.
|
||||
*/
|
||||
async startPlayback(request: SpotifyPlayRequest): Promise<boolean> {
|
||||
const client = this.deps.client;
|
||||
const playback = this.playback;
|
||||
const ok = await this.command(async () => {
|
||||
try {
|
||||
await client.play(request);
|
||||
} catch (error) {
|
||||
if (!(error instanceof SpotifyApiError) || error.reason !== "NO_ACTIVE_DEVICE") throw error;
|
||||
const device = pickDevice(await client.devices());
|
||||
if (!device) throw error;
|
||||
await client.play(request, device.id);
|
||||
}
|
||||
}, () => {
|
||||
this.lastPlayingAtMs = this.now();
|
||||
if (playback && this.playback === playback && !request.contextUri && !request.uris) {
|
||||
playback.playing = true;
|
||||
this.fetchedAtMs = this.now();
|
||||
}
|
||||
});
|
||||
return ok;
|
||||
}
|
||||
|
||||
/** The account changed (signed in, out, or as someone else): forget the old account's state. */
|
||||
accountChanged(): void {
|
||||
this.playback = null;
|
||||
this.upcoming = [];
|
||||
this.queueCurrentUri = "";
|
||||
this.art = null;
|
||||
this.artVersion++;
|
||||
this.liked = null;
|
||||
this.lastPlayingAtMs = 0;
|
||||
this.emit(true);
|
||||
if (this.listeners.size) this.schedulePoll(0);
|
||||
}
|
||||
|
||||
/** Show a message about a failed command in the state's notice for a few seconds. */
|
||||
showNotice(message: string): void {
|
||||
this.notice = message;
|
||||
this.noticeUntilMs = this.now() + NOTICE_MS;
|
||||
if (this.noticeTimer !== null) clearTimeout(this.noticeTimer);
|
||||
this.noticeTimer = setTimeout(() => {
|
||||
this.noticeTimer = null;
|
||||
this.emit(true);
|
||||
}, NOTICE_MS + 50);
|
||||
this.emit(true);
|
||||
}
|
||||
|
||||
private currentNotice(): string {
|
||||
return this.notice && this.now() < this.noticeUntilMs ? this.notice : "";
|
||||
}
|
||||
|
||||
private customActions(playback: SpotifyPlayback): MediaCustomAction[] {
|
||||
if (!playback.item) return [];
|
||||
const actions: MediaCustomAction[] = [];
|
||||
if (!playback.disallows.includes("toggling_shuffle")) {
|
||||
actions.push({ action: "shuffle", name: playback.shuffle ? "Shuffle: on" : "Shuffle: off" });
|
||||
}
|
||||
if (!playback.disallows.includes("toggling_repeat_context") || !playback.disallows.includes("toggling_repeat_track")) {
|
||||
actions.push({ action: "repeat", name: REPEAT_LABEL[playback.repeat] });
|
||||
}
|
||||
const liked = this.liked;
|
||||
if (playback.item.type === "track" && liked?.uri === playback.item.uri && liked.value !== null) {
|
||||
actions.push({ action: "like", name: liked.value ? "Unlike song" : "Like song" });
|
||||
}
|
||||
return actions;
|
||||
}
|
||||
|
||||
/** Run a command; on success apply its expected effect at once, then re-read to confirm. */
|
||||
private async command(run: () => Promise<void>, applied?: () => void): Promise<boolean> {
|
||||
let ok = false;
|
||||
try {
|
||||
await run();
|
||||
ok = true;
|
||||
applied?.();
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
this.showNotice(errorMessage(error));
|
||||
}
|
||||
this.settlePolls = SETTLE_POLLS;
|
||||
this.schedulePoll(SETTLE_POLL_MS);
|
||||
return ok;
|
||||
}
|
||||
|
||||
private async sendVolume(deviceId: string): Promise<void> {
|
||||
this.volumeSending = true;
|
||||
try {
|
||||
while (this.volumeTarget !== null) {
|
||||
const volume = this.volumeTarget;
|
||||
try {
|
||||
await this.deps.client.setVolume(volume, deviceId);
|
||||
} catch (error) {
|
||||
this.showNotice(errorMessage(error));
|
||||
}
|
||||
if (this.volumeTarget === volume) this.volumeTarget = null;
|
||||
}
|
||||
} finally {
|
||||
this.volumeSending = false;
|
||||
}
|
||||
}
|
||||
|
||||
private poll(): Promise<void> {
|
||||
if (this.polling) {
|
||||
this.pollAgain = true;
|
||||
return this.polling;
|
||||
}
|
||||
this.polling = this.pollOnce().finally(() => {
|
||||
this.polling = null;
|
||||
if (this.pollAgain) {
|
||||
this.pollAgain = false;
|
||||
void this.poll();
|
||||
}
|
||||
});
|
||||
return this.polling;
|
||||
}
|
||||
|
||||
private async pollOnce(): Promise<void> {
|
||||
this.clearPollTimer();
|
||||
if (!this.deps.isSignedIn()) {
|
||||
if (this.playback) {
|
||||
this.playback = null;
|
||||
this.emit(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let delay: number;
|
||||
try {
|
||||
const playback = await this.deps.client.playback();
|
||||
this.fetchedAtMs = this.now();
|
||||
this.applyPlayback(playback);
|
||||
delay = this.nextPollDelay(playback);
|
||||
} catch (error) {
|
||||
if (!(error instanceof SpotifyApiError) || error.reason !== "signed_out") {
|
||||
console.warn(`[spotify] playback read failed: ${error}`);
|
||||
}
|
||||
// Revoked access signs out (SpotifySession); the account listener resets the state.
|
||||
delay = POLL_ERROR_MS;
|
||||
}
|
||||
if (this.settlePolls > 0) {
|
||||
this.settlePolls--;
|
||||
delay = Math.min(delay, 1_500);
|
||||
}
|
||||
this.emit(false);
|
||||
this.schedulePoll(delay);
|
||||
}
|
||||
|
||||
private applyPlayback(playback: SpotifyPlayback | null): void {
|
||||
const previousUri = this.playback?.item?.uri ?? "";
|
||||
this.playback = playback;
|
||||
if (playback?.playing) this.lastPlayingAtMs = this.now();
|
||||
const item = playback?.item ?? null;
|
||||
if ((item?.uri ?? "") !== previousUri) this.itemChanged(item);
|
||||
}
|
||||
|
||||
/** A new item: fetch its art and whether it is liked. */
|
||||
private itemChanged(item: SpotifyItem | null): void {
|
||||
const url = item ? pickArtUrl(item) : "";
|
||||
if (url !== (this.art?.url ?? "")) {
|
||||
this.artVersion++;
|
||||
this.art = url ? { url, bytes: null, decoded: new Map() } : null;
|
||||
if (url) void this.loadArt(url);
|
||||
}
|
||||
this.liked = item?.type === "track" ? { uri: item.uri, value: null } : null;
|
||||
if (item?.type === "track") void this.loadLiked(item.uri);
|
||||
}
|
||||
|
||||
private async loadArt(url: string): Promise<void> {
|
||||
let bytes: Uint8Array | null = null;
|
||||
try {
|
||||
bytes = await this.deps.download(url);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] album art download failed: ${error}`);
|
||||
}
|
||||
if (!bytes?.length || this.art?.url !== url) return;
|
||||
this.art.bytes = bytes;
|
||||
this.artVersion++;
|
||||
this.emit(true);
|
||||
}
|
||||
|
||||
private async loadLiked(uri: string): Promise<void> {
|
||||
try {
|
||||
const [value] = await this.deps.client.libraryContains([uri]);
|
||||
if (this.liked?.uri !== uri) return;
|
||||
this.liked.value = Boolean(value);
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] library check failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadQueue(): Promise<void> {
|
||||
this.queueLoading = true;
|
||||
try {
|
||||
const queue = await this.deps.client.queue();
|
||||
this.upcoming = queue.upcoming;
|
||||
this.queueCurrentUri = queue.current?.uri ?? "";
|
||||
this.queueFetchedAtMs = this.now();
|
||||
this.emit(true);
|
||||
} catch (error) {
|
||||
console.warn(`[spotify] queue read failed: ${error}`);
|
||||
this.queueFetchedAtMs = this.now();
|
||||
} finally {
|
||||
this.queueLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
private nextPollDelay(playback: SpotifyPlayback | null): number {
|
||||
if (!playback) return POLL_IDLE_MS;
|
||||
if (!playback.playing) return POLL_PAUSED_MS;
|
||||
const durationMs = playback.item?.durationMs ?? -1;
|
||||
// Catch the change to the next track as it happens.
|
||||
const remaining = durationMs > 0 ? durationMs - playback.progressMs + 700 : POLL_PLAYING_MS;
|
||||
return Math.max(1_000, Math.min(POLL_PLAYING_MS, remaining));
|
||||
}
|
||||
|
||||
private schedulePoll(delayMs: number): void {
|
||||
if (!this.listeners.size && !this.settlePolls) return;
|
||||
this.clearPollTimer();
|
||||
this.pollTimer = setTimeout(() => {
|
||||
this.pollTimer = null;
|
||||
void this.poll();
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
private clearPollTimer(): void {
|
||||
if (this.pollTimer !== null) clearTimeout(this.pollTimer);
|
||||
this.pollTimer = null;
|
||||
}
|
||||
|
||||
/** Tell listeners; unforced, only when something they draw (other than the advancing position) changed. */
|
||||
private emit(force: boolean): void {
|
||||
if (!this.listeners.size) return;
|
||||
const state = this.snapshot();
|
||||
const signature = JSON.stringify({ ...state, positionMs: 0 });
|
||||
if (!force && signature === this.lastSignature) return;
|
||||
this.lastSignature = signature;
|
||||
for (const listener of this.listeners) listener(state);
|
||||
}
|
||||
}
|
||||
|
||||
/** "Spotify", or "Spotify · <device>" when it plays elsewhere than a phone. */
|
||||
export function spotifyAppName(device: SpotifyDevice | null): string {
|
||||
if (!device?.name || device.type === "Smartphone") return SPOTIFY_APP_NAME;
|
||||
return `${SPOTIFY_APP_NAME} · ${device.name}`;
|
||||
}
|
||||
|
||||
/** Spotify sets a speaker's or computer's volume; a phone's (this one) goes through the system volume. */
|
||||
function usesDeviceVolume(device: SpotifyDevice | null): boolean {
|
||||
return Boolean(device && device.type !== "Smartphone" && device.supportsVolume && device.volumePercent >= 0 && device.id);
|
||||
}
|
||||
|
||||
/** A device to start playback on when none is active: an open one, a phone first. */
|
||||
export function pickDevice(devices: SpotifyDevice[]): SpotifyDevice | null {
|
||||
const usable = devices.filter((device) => device.id && !device.restricted);
|
||||
return usable.find((device) => device.active)
|
||||
?? usable.find((device) => device.type === "Smartphone")
|
||||
?? usable[0]
|
||||
?? null;
|
||||
}
|
||||
|
||||
function pickArtUrl(item: SpotifyItem): string {
|
||||
const images = [...item.images].sort((a, b) => a.width - b.width);
|
||||
return (images.find((image) => image.width >= ART_MIN_WIDTH) ?? images[images.length - 1])?.url ?? "";
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
return String(error);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import {
|
||||
readSpotifyAccount,
|
||||
saveSpotifyAccount,
|
||||
signInToSpotify,
|
||||
SPOTIFY_ACCOUNT_KEY,
|
||||
SPOTIFY_CLIENT_ID_KEY,
|
||||
SpotifySession,
|
||||
type SpotifyAccount,
|
||||
type WebSignIn,
|
||||
} from "./spotify-auth";
|
||||
import { SpotifyLibrary } from "./spotify-library";
|
||||
import { errorMessage, SpotifyPlayer } from "./spotify-player";
|
||||
import { SpotifyClient, type SpotifyFetch, type SpotifySearchType } from "./spotify-web-api";
|
||||
|
||||
/** Give up on a sign-in nobody finished on the phone. */
|
||||
const SIGN_IN_TIMEOUT_MS = 10 * 60_000;
|
||||
|
||||
/** What the platform supplies (spotify.ios.ts, spotify.ts). */
|
||||
export type SpotifyPlatform = {
|
||||
/** Spotify's app id on this platform (SPOTIFY_IOS_BUNDLE_ID / SPOTIFY_ANDROID_PACKAGE). */
|
||||
packageName: string;
|
||||
fetch: SpotifyFetch;
|
||||
webSignIn: WebSignIn;
|
||||
download(url: string): Promise<Uint8Array | null>;
|
||||
decodeArt(bytes: Uint8Array, maxSize: number): GrayImage | null;
|
||||
systemVolume: { get(): number; set(percent: number): void };
|
||||
};
|
||||
|
||||
export type SpotifySignInState =
|
||||
| { phase: "idle"; error: string }
|
||||
/** The sign-in sheet is open on the phone, or waiting for Faceclaw to come to the foreground. */
|
||||
| { phase: "waiting"; error: "" };
|
||||
|
||||
/**
|
||||
* Spotify for the Music app, the media bridges and the assistant: the
|
||||
* account and its sign-in, the Web API client, the player and the library.
|
||||
*/
|
||||
export class SpotifyService {
|
||||
readonly client: SpotifyClient;
|
||||
readonly player: SpotifyPlayer;
|
||||
readonly library: SpotifyLibrary;
|
||||
private signInState: SpotifySignInState = { phase: "idle", error: "" };
|
||||
private readonly listeners = new Set<() => void>();
|
||||
private accountKey: string;
|
||||
|
||||
constructor(private readonly platform: SpotifyPlatform) {
|
||||
const isSignedIn = () => readSpotifyAccount() !== null;
|
||||
this.client = new SpotifyClient(platform.fetch, new SpotifySession(platform.fetch));
|
||||
this.player = new SpotifyPlayer({
|
||||
client: this.client,
|
||||
packageName: platform.packageName,
|
||||
isSignedIn,
|
||||
download: (url) => platform.download(url),
|
||||
decodeArt: (bytes, maxSize) => platform.decodeArt(bytes, maxSize),
|
||||
systemVolume: platform.systemVolume,
|
||||
});
|
||||
this.library = new SpotifyLibrary({
|
||||
client: this.client,
|
||||
packageName: platform.packageName,
|
||||
player: this.player,
|
||||
isSignedIn,
|
||||
userId: () => readSpotifyAccount()?.userId ?? "",
|
||||
});
|
||||
this.accountKey = accountIdentity(readSpotifyAccount());
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key !== SPOTIFY_ACCOUNT_KEY && key !== SPOTIFY_CLIENT_ID_KEY) return;
|
||||
// Token refreshes rewrite the account too; only a different sign-in resets the player.
|
||||
const identity = accountIdentity(readSpotifyAccount());
|
||||
if (identity !== this.accountKey) {
|
||||
this.accountKey = identity;
|
||||
this.player.accountChanged();
|
||||
}
|
||||
this.emit();
|
||||
});
|
||||
}
|
||||
|
||||
account(): SpotifyAccount | null {
|
||||
return readSpotifyAccount();
|
||||
}
|
||||
|
||||
signInStatus(): SpotifySignInState {
|
||||
return this.signInState;
|
||||
}
|
||||
|
||||
/** Repaint hook for the sign-in menu: account or sign-in progress changed. */
|
||||
onChange(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open Spotify's sign-in page on the phone (it waits for Faceclaw to be in
|
||||
* the foreground); the outcome lands in signInStatus().
|
||||
*/
|
||||
async signIn(): Promise<void> {
|
||||
if (this.signInState.phase === "waiting") return;
|
||||
this.signInState = { phase: "waiting", error: "" };
|
||||
this.emit();
|
||||
const timer = setTimeout(() => this.platform.webSignIn.cancel(), SIGN_IN_TIMEOUT_MS);
|
||||
try {
|
||||
await signInToSpotify(this.platform.fetch, this.platform.webSignIn);
|
||||
this.signInState = { phase: "idle", error: "" };
|
||||
} catch (error) {
|
||||
const message = errorMessage(error);
|
||||
this.signInState = { phase: "idle", error: message === "cancelled" ? "" : message };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
this.emit();
|
||||
}
|
||||
}
|
||||
|
||||
cancelSignIn(): void {
|
||||
this.platform.webSignIn.cancel();
|
||||
}
|
||||
|
||||
signOut(): void {
|
||||
saveSpotifyAccount(null);
|
||||
this.signInState = { phase: "idle", error: "" };
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Spotify and play the best match: a track, or the album, artist or
|
||||
* playlist when the request names that kind. Resolves to what started, or
|
||||
* rejects with why not.
|
||||
*/
|
||||
async playSearch(query: string, kind: SpotifySearchType = "track"): Promise<string> {
|
||||
if (!readSpotifyAccount()) throw new Error("Not signed in to Spotify. Sign in under Spotify in the Music app menu on the glasses.");
|
||||
const results = await this.client.search(query, [kind], 3);
|
||||
let request: { contextUri?: string; uris?: string[] } | null = null;
|
||||
let description = "";
|
||||
if (kind === "track" && results.tracks[0]) {
|
||||
const track = results.tracks[0];
|
||||
request = { uris: [track.uri] };
|
||||
description = `${track.name}${track.artists ? ` by ${track.artists}` : ""}`;
|
||||
} else if (kind === "album" && results.albums[0]) {
|
||||
const album = results.albums[0];
|
||||
request = { contextUri: album.uri };
|
||||
description = `the album ${album.name}${album.artists ? ` by ${album.artists}` : ""}`;
|
||||
} else if (kind === "artist" && results.artists[0]) {
|
||||
request = { contextUri: results.artists[0].uri };
|
||||
description = results.artists[0].name;
|
||||
} else if (kind === "playlist" && results.playlists[0]) {
|
||||
request = { contextUri: results.playlists[0].uri };
|
||||
description = `the playlist ${results.playlists[0].name}`;
|
||||
}
|
||||
if (!request) throw new Error(`Spotify found no ${kind} matching "${query}".`);
|
||||
if (!(await this.player.startPlayback(request))) {
|
||||
throw new Error(this.player.snapshot().notice || "Spotify didn't start playback.");
|
||||
}
|
||||
return description;
|
||||
}
|
||||
|
||||
private emit(): void {
|
||||
for (const listener of this.listeners) listener();
|
||||
}
|
||||
}
|
||||
|
||||
function accountIdentity(account: SpotifyAccount | null): string {
|
||||
return account ? `${account.clientId}/${account.userId}` : "";
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* Spotify Web API over HTTPS with the signed-in user's token: the player
|
||||
* (what is playing on any of the user's Spotify Connect devices, and control
|
||||
* of it), the user's library, and search, normalized into the small shapes
|
||||
* the Music app uses. No NativeScript imports, so the Node tests drive it with
|
||||
* a fake fetch.
|
||||
*
|
||||
* Every personal client ID is a development-mode app, which since February
|
||||
* 2026 means: the client ID's owner needs Premium, playback control needs
|
||||
* Premium, at most five allowlisted users, only owned or collaborative
|
||||
* playlists list their items, no batch lookups, and search and artist albums
|
||||
* return at most 10 results per page.
|
||||
*/
|
||||
|
||||
export const SPOTIFY_API_BASE = "https://api.spotify.com/v1";
|
||||
|
||||
const HTTP_TIMEOUT_MS = 12_000;
|
||||
/** The most a library listing gathers (a page is 50); enough to scroll through on the glasses. */
|
||||
const MAX_LIST_ITEMS = 500;
|
||||
|
||||
/** The subset of fetch this client needs (fetchTextWithUserAgent in the app). */
|
||||
export type SpotifyFetch = (
|
||||
url: string,
|
||||
init: { method: string; headers: Record<string, string>; body?: string },
|
||||
) => Promise<{ status: number; text(): Promise<string> }>;
|
||||
|
||||
/** Hands out a current access token, refreshing it when expired or when forced (after a 401). */
|
||||
export type SpotifyTokenSource = {
|
||||
accessToken(forceRefresh: boolean): Promise<string>;
|
||||
};
|
||||
|
||||
/** A failed request; `reason` is Spotify's player error code (NO_ACTIVE_DEVICE, PREMIUM_REQUIRED, ...). */
|
||||
export class SpotifyApiError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly status: number,
|
||||
readonly reason = "",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SpotifyApiError";
|
||||
}
|
||||
}
|
||||
|
||||
export type SpotifyImage = { url: string; width: number; height: number };
|
||||
|
||||
export type SpotifyItem = {
|
||||
uri: string;
|
||||
type: "track" | "episode";
|
||||
name: string;
|
||||
/** Track artists, comma-joined; for an episode, its show. */
|
||||
artists: string;
|
||||
/** The track's album; for an episode, its show. "" when the API leaves it out (album track listings). */
|
||||
album: string;
|
||||
durationMs: number;
|
||||
images: SpotifyImage[];
|
||||
};
|
||||
|
||||
export type SpotifyDevice = {
|
||||
id: string;
|
||||
name: string;
|
||||
/** "Smartphone", "Computer", "Speaker", ... */
|
||||
type: string;
|
||||
active: boolean;
|
||||
/** Restricted devices accept no Web API commands. */
|
||||
restricted: boolean;
|
||||
/** -1 when the device reports none. */
|
||||
volumePercent: number;
|
||||
supportsVolume: boolean;
|
||||
};
|
||||
|
||||
export type SpotifyRepeat = "off" | "context" | "track";
|
||||
|
||||
export type SpotifyPlayback = {
|
||||
device: SpotifyDevice | null;
|
||||
playing: boolean;
|
||||
progressMs: number;
|
||||
shuffle: boolean;
|
||||
repeat: SpotifyRepeat;
|
||||
/** The playlist / album / artist playing from; "" for none (a queue of single tracks). */
|
||||
contextUri: string;
|
||||
/** null during an ad, between items, or for content the API does not describe. */
|
||||
item: SpotifyItem | null;
|
||||
/** "track", "episode", "ad" or "unknown". */
|
||||
playingType: string;
|
||||
/** The player actions the context forbids, as Spotify names them: pausing, resuming, skipping_next, ... */
|
||||
disallows: string[];
|
||||
};
|
||||
|
||||
export type SpotifyQueue = { current: SpotifyItem | null; upcoming: SpotifyItem[] };
|
||||
|
||||
export type SpotifyUser = { id: string; displayName: string; product: string };
|
||||
|
||||
export type SpotifyPlaylist = {
|
||||
id: string;
|
||||
uri: string;
|
||||
name: string;
|
||||
ownerId: string;
|
||||
ownerName: string;
|
||||
collaborative: boolean;
|
||||
total: number;
|
||||
};
|
||||
|
||||
export type SpotifyAlbum = { id: string; uri: string; name: string; artists: string };
|
||||
|
||||
export type SpotifyArtist = { id: string; uri: string; name: string };
|
||||
|
||||
/** A played track and the context it played from ("" when none). */
|
||||
export type SpotifyPlayed = { item: SpotifyItem; contextUri: string };
|
||||
|
||||
export type SpotifySearchResults = {
|
||||
tracks: SpotifyItem[];
|
||||
albums: SpotifyAlbum[];
|
||||
artists: SpotifyArtist[];
|
||||
playlists: SpotifyPlaylist[];
|
||||
};
|
||||
|
||||
export type SpotifySearchType = "track" | "album" | "artist" | "playlist";
|
||||
|
||||
/**
|
||||
* What to play: a context (playlist, album, artist, show) optionally starting
|
||||
* at one of its items, or a list of track URIs. Empty resumes the current
|
||||
* playback.
|
||||
*/
|
||||
export type SpotifyPlayRequest = {
|
||||
contextUri?: string;
|
||||
uris?: string[];
|
||||
offsetUri?: string;
|
||||
offsetPosition?: number;
|
||||
};
|
||||
|
||||
type Query = Record<string, string | number | boolean | undefined>;
|
||||
|
||||
export class SpotifyClient {
|
||||
constructor(
|
||||
private readonly fetchImpl: SpotifyFetch,
|
||||
private readonly tokens: SpotifyTokenSource,
|
||||
) {}
|
||||
|
||||
/** The current playback on whichever device is active; null when nothing is. */
|
||||
async playback(): Promise<SpotifyPlayback | null> {
|
||||
const raw = await this.request("GET", "/me/player", { query: { additional_types: "episode" } });
|
||||
return raw ? parsePlayback(raw) : null;
|
||||
}
|
||||
|
||||
async queue(): Promise<SpotifyQueue> {
|
||||
const raw = await this.request("GET", "/me/player/queue");
|
||||
return {
|
||||
current: parseItem(raw?.currently_playing),
|
||||
upcoming: asArray(raw?.queue).map(parseItem).filter(isPresent),
|
||||
};
|
||||
}
|
||||
|
||||
async devices(): Promise<SpotifyDevice[]> {
|
||||
const raw = await this.request("GET", "/me/player/devices");
|
||||
return asArray(raw?.devices).map(parseDevice).filter(isPresent);
|
||||
}
|
||||
|
||||
async play(request: SpotifyPlayRequest = {}, deviceId = ""): Promise<void> {
|
||||
const body: Record<string, unknown> = {};
|
||||
if (request.contextUri) body.context_uri = request.contextUri;
|
||||
if (request.uris?.length) body.uris = request.uris;
|
||||
if (request.offsetUri) body.offset = { uri: request.offsetUri };
|
||||
else if (request.offsetPosition !== undefined) body.offset = { position: request.offsetPosition };
|
||||
await this.request("PUT", "/me/player/play", { query: { device_id: deviceId || undefined }, body });
|
||||
}
|
||||
|
||||
async pause(): Promise<void> {
|
||||
await this.request("PUT", "/me/player/pause");
|
||||
}
|
||||
|
||||
async next(): Promise<void> {
|
||||
await this.request("POST", "/me/player/next");
|
||||
}
|
||||
|
||||
async previous(): Promise<void> {
|
||||
await this.request("POST", "/me/player/previous");
|
||||
}
|
||||
|
||||
async setShuffle(on: boolean): Promise<void> {
|
||||
await this.request("PUT", "/me/player/shuffle", { query: { state: on } });
|
||||
}
|
||||
|
||||
async setRepeat(mode: SpotifyRepeat): Promise<void> {
|
||||
await this.request("PUT", "/me/player/repeat", { query: { state: mode } });
|
||||
}
|
||||
|
||||
async setVolume(percent: number, deviceId = ""): Promise<void> {
|
||||
const volume = Math.max(0, Math.min(100, Math.round(percent)));
|
||||
await this.request("PUT", "/me/player/volume", { query: { volume_percent: volume, device_id: deviceId || undefined } });
|
||||
}
|
||||
|
||||
/** Move playback to a device (one that shows in devices() but is not active), optionally playing. */
|
||||
async transfer(deviceId: string, play: boolean): Promise<void> {
|
||||
await this.request("PUT", "/me/player", { body: { device_ids: [deviceId], play } });
|
||||
}
|
||||
|
||||
async me(): Promise<SpotifyUser> {
|
||||
const raw = await this.request("GET", "/me");
|
||||
return { id: str(raw?.id), displayName: str(raw?.display_name) || str(raw?.id), product: str(raw?.product) };
|
||||
}
|
||||
|
||||
/** Whether each URI is in the user's library (Liked Songs for tracks). */
|
||||
async libraryContains(uris: string[]): Promise<boolean[]> {
|
||||
const raw = await this.request("GET", "/me/library/contains", { query: { uris: uris.join(",") } });
|
||||
return asArray(raw).map(Boolean);
|
||||
}
|
||||
|
||||
async saveToLibrary(uris: string[]): Promise<void> {
|
||||
await this.request("PUT", "/me/library", { query: { uris: uris.join(",") } });
|
||||
}
|
||||
|
||||
async removeFromLibrary(uris: string[]): Promise<void> {
|
||||
await this.request("DELETE", "/me/library", { query: { uris: uris.join(",") } });
|
||||
}
|
||||
|
||||
playlists(): Promise<SpotifyPlaylist[]> {
|
||||
return this.collect("/me/playlists", { limit: 50 }, (raw) => asArray(raw?.items).map(parsePlaylist));
|
||||
}
|
||||
|
||||
/** Only playlists the user owns or collaborates on list their items (403 otherwise). */
|
||||
playlistItems(playlistId: string): Promise<SpotifyItem[]> {
|
||||
return this.collect(`/playlists/${encodeURIComponent(playlistId)}/items`, { limit: 50, additional_types: "track,episode" },
|
||||
(raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.item ?? entry?.track)));
|
||||
}
|
||||
|
||||
savedTracks(): Promise<SpotifyItem[]> {
|
||||
return this.collect("/me/tracks", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseItem(entry?.track)));
|
||||
}
|
||||
|
||||
savedAlbums(): Promise<SpotifyAlbum[]> {
|
||||
return this.collect("/me/albums", { limit: 50 }, (raw) => asArray(raw?.items).map((entry: any) => parseAlbum(entry?.album)));
|
||||
}
|
||||
|
||||
albumTracks(albumId: string): Promise<SpotifyItem[]> {
|
||||
return this.collect(`/albums/${encodeURIComponent(albumId)}/tracks`, { limit: 50 }, (raw) => asArray(raw?.items).map(parseItem));
|
||||
}
|
||||
|
||||
followedArtists(): Promise<SpotifyArtist[]> {
|
||||
// Cursor-paged, with the page wrapped in an "artists" object.
|
||||
return this.collect("/me/following", { type: "artist", limit: 50 }, (raw) => asArray(raw?.artists?.items).map(parseArtist),
|
||||
(raw) => str(raw?.artists?.next));
|
||||
}
|
||||
|
||||
artistAlbums(artistId: string): Promise<SpotifyAlbum[]> {
|
||||
return this.collect(`/artists/${encodeURIComponent(artistId)}/albums`, { include_groups: "album,single,compilation", limit: 10 },
|
||||
(raw) => asArray(raw?.items).map(parseAlbum), undefined, 100);
|
||||
}
|
||||
|
||||
/** Recently played tracks, newest first (one page: the API keeps only the last 50). */
|
||||
async recentlyPlayed(): Promise<SpotifyPlayed[]> {
|
||||
const raw = await this.request("GET", "/me/player/recently-played", { query: { limit: 50 } });
|
||||
return asArray(raw?.items)
|
||||
.map((entry: any) => {
|
||||
const item = parseItem(entry?.track);
|
||||
return item ? { item, contextUri: str(entry?.context?.uri) } : null;
|
||||
})
|
||||
.filter(isPresent);
|
||||
}
|
||||
|
||||
async search(query: string, types: SpotifySearchType[], limit = 5): Promise<SpotifySearchResults> {
|
||||
const raw = await this.request("GET", "/search", {
|
||||
query: { q: query, type: types.join(","), limit: Math.max(1, Math.min(10, Math.round(limit))) },
|
||||
});
|
||||
return {
|
||||
tracks: asArray(raw?.tracks?.items).map(parseItem).filter(isPresent),
|
||||
albums: asArray(raw?.albums?.items).map(parseAlbum).filter(isPresent),
|
||||
artists: asArray(raw?.artists?.items).map(parseArtist).filter(isPresent),
|
||||
// Search results include nulls for playlists the API can no longer return.
|
||||
playlists: asArray(raw?.playlists?.items).map(parsePlaylist).filter(isPresent),
|
||||
};
|
||||
}
|
||||
|
||||
/** Gather a paged listing by following its `next` links, up to maxItems. */
|
||||
private async collect<T>(
|
||||
path: string,
|
||||
query: Query,
|
||||
parse: (raw: any) => Array<T | null>,
|
||||
nextOf: (raw: any) => string = (raw) => str(raw?.next),
|
||||
maxItems = MAX_LIST_ITEMS,
|
||||
): Promise<T[]> {
|
||||
const items: T[] = [];
|
||||
let raw = await this.request("GET", path, { query });
|
||||
for (;;) {
|
||||
for (const item of parse(raw)) if (item) items.push(item);
|
||||
const next = nextOf(raw);
|
||||
if (!next || items.length >= maxItems) return items.slice(0, maxItems);
|
||||
raw = await this.request("GET", next);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One API call. `path` is relative to the API base, or an absolute `next`
|
||||
* link from a page. Resolves to the parsed JSON body (null when empty);
|
||||
* a 401 refreshes the token and retries once.
|
||||
*/
|
||||
private async request(method: string, path: string, options: { query?: Query; body?: unknown } = {}): Promise<any> {
|
||||
const url = path.startsWith("https://") ? path : `${SPOTIFY_API_BASE}${path}${buildQuery(options.query)}`;
|
||||
const headers: Record<string, string> = { Accept: "application/json" };
|
||||
if (options.body !== undefined) headers["Content-Type"] = "application/json";
|
||||
// A command without a body still sends an empty one: Spotify answers a
|
||||
// PUT or POST with no Content-Length with 411.
|
||||
const body = options.body !== undefined ? JSON.stringify(options.body) : method === "GET" ? undefined : "";
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const authorization = `Bearer ${await this.tokens.accessToken(attempt > 0)}`;
|
||||
const response = await withTimeout(this.fetchImpl(url, { method, headers: { ...headers, Authorization: authorization }, body }));
|
||||
const text = await withTimeout(response.text());
|
||||
if (response.status === 401 && attempt === 0) continue;
|
||||
if (response.status >= 200 && response.status < 300) {
|
||||
if (!text.trim()) return null;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
// Commands answer 200 with a snapshot id or plain text; nothing to read.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
throw apiError(response.status, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Form / query encoding (not URLSearchParams, which the NativeScript runtime mishandles). */
|
||||
export function buildQuery(query: Query | undefined): string {
|
||||
if (!query) return "";
|
||||
const parts = Object.keys(query)
|
||||
.filter((key) => query[key] !== undefined && query[key] !== "")
|
||||
.map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(String(query[key]))}`);
|
||||
return parts.length ? `?${parts.join("&")}` : "";
|
||||
}
|
||||
|
||||
async function withTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const timeout = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new SpotifyApiError("Spotify didn't answer. Check the phone's connection.", 0)), HTTP_TIMEOUT_MS);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} catch (error) {
|
||||
if (error instanceof SpotifyApiError) throw error;
|
||||
throw new SpotifyApiError(`Can't reach Spotify: ${(error as Error)?.message ?? error}`, 0);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** A message for the glasses from Spotify's error body ({error: {status, message, reason}}). */
|
||||
function apiError(status: number, body: string): SpotifyApiError {
|
||||
let message = "";
|
||||
let reason = "";
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
message = str(parsed?.error?.message ?? parsed?.error_description);
|
||||
reason = str(parsed?.error?.reason);
|
||||
} catch {
|
||||
// Not JSON (a gateway error page).
|
||||
}
|
||||
switch (reason) {
|
||||
case "NO_ACTIVE_DEVICE":
|
||||
return new SpotifyApiError("Spotify isn't open on any device. Open Spotify on your phone, then try again.", status, reason);
|
||||
case "PREMIUM_REQUIRED":
|
||||
return new SpotifyApiError("Controlling Spotify playback needs Spotify Premium.", status, reason);
|
||||
}
|
||||
if (status === 401) return new SpotifyApiError("Spotify no longer accepts this sign-in. Sign in again from the Music app menu.", status, reason);
|
||||
if (status === 403 && /registered|developer dashboard/i.test(message)) {
|
||||
return new SpotifyApiError("This Spotify account isn't on the client ID's user list (Developer Dashboard > User Management).", status, reason);
|
||||
}
|
||||
if (status === 429) return new SpotifyApiError("Spotify is limiting requests; try again in a minute.", status, reason);
|
||||
return new SpotifyApiError(message ? `Spotify: ${message}` : `Spotify error (HTTP ${status}).`, status, reason);
|
||||
}
|
||||
|
||||
export function parsePlayback(raw: any): SpotifyPlayback {
|
||||
const repeat = str(raw?.repeat_state);
|
||||
const disallows = raw?.actions?.disallows;
|
||||
return {
|
||||
device: parseDevice(raw?.device),
|
||||
playing: raw?.is_playing === true,
|
||||
progressMs: num(raw?.progress_ms, 0),
|
||||
shuffle: raw?.shuffle_state === true,
|
||||
repeat: repeat === "context" || repeat === "track" ? repeat : "off",
|
||||
contextUri: str(raw?.context?.uri),
|
||||
item: parseItem(raw?.item),
|
||||
playingType: str(raw?.currently_playing_type) || "unknown",
|
||||
disallows: disallows && typeof disallows === "object" ? Object.keys(disallows).filter((key) => disallows[key] === true) : [],
|
||||
};
|
||||
}
|
||||
|
||||
export function parseItem(raw: any): SpotifyItem | null {
|
||||
if (!raw || typeof raw !== "object" || typeof raw.uri !== "string" || !raw.uri) return null;
|
||||
if (raw.type === "episode") {
|
||||
const show = str(raw.show?.name);
|
||||
return {
|
||||
uri: raw.uri,
|
||||
type: "episode",
|
||||
name: str(raw.name),
|
||||
artists: show || str(raw.show?.publisher),
|
||||
album: show,
|
||||
durationMs: num(raw.duration_ms, -1),
|
||||
images: parseImages(raw.images?.length ? raw.images : raw.show?.images),
|
||||
};
|
||||
}
|
||||
return {
|
||||
uri: raw.uri,
|
||||
type: "track",
|
||||
name: str(raw.name),
|
||||
artists: joinNames(raw.artists),
|
||||
album: str(raw.album?.name),
|
||||
durationMs: num(raw.duration_ms, -1),
|
||||
images: parseImages(raw.album?.images),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseDevice(raw: any): SpotifyDevice | null {
|
||||
if (!raw || typeof raw !== "object") return null;
|
||||
const volume = raw.volume_percent;
|
||||
return {
|
||||
id: str(raw.id),
|
||||
name: str(raw.name),
|
||||
type: str(raw.type),
|
||||
active: raw.is_active === true,
|
||||
restricted: raw.is_restricted === true,
|
||||
volumePercent: typeof volume === "number" && Number.isFinite(volume) ? volume : -1,
|
||||
supportsVolume: raw.supports_volume === true,
|
||||
};
|
||||
}
|
||||
|
||||
function parsePlaylist(raw: any): SpotifyPlaylist | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return {
|
||||
id: str(raw.id),
|
||||
uri: str(raw.uri) || `spotify:playlist:${raw.id}`,
|
||||
name: str(raw.name),
|
||||
ownerId: str(raw.owner?.id),
|
||||
ownerName: str(raw.owner?.display_name) || str(raw.owner?.id),
|
||||
collaborative: raw.collaborative === true,
|
||||
// `items` replaced the deprecated `tracks` summary in 2026.
|
||||
total: num(raw.items?.total ?? raw.tracks?.total, -1),
|
||||
};
|
||||
}
|
||||
|
||||
function parseAlbum(raw: any): SpotifyAlbum | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return { id: str(raw.id), uri: str(raw.uri) || `spotify:album:${raw.id}`, name: str(raw.name), artists: joinNames(raw.artists) };
|
||||
}
|
||||
|
||||
function parseArtist(raw: any): SpotifyArtist | null {
|
||||
if (!raw || typeof raw !== "object" || !raw.id) return null;
|
||||
return { id: str(raw.id), uri: str(raw.uri) || `spotify:artist:${raw.id}`, name: str(raw.name) };
|
||||
}
|
||||
|
||||
function parseImages(raw: any): SpotifyImage[] {
|
||||
return asArray(raw)
|
||||
.filter((image: any) => image && typeof image.url === "string" && image.url)
|
||||
.map((image: any) => ({ url: image.url, width: num(image.width, 0), height: num(image.height, 0) }));
|
||||
}
|
||||
|
||||
function joinNames(raw: any): string {
|
||||
return asArray(raw).map((entry: any) => str(entry?.name)).filter(Boolean).join(", ");
|
||||
}
|
||||
|
||||
function asArray(value: unknown): any[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function str(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function num(value: unknown, fallback: number): number {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
||||
}
|
||||
|
||||
function isPresent<T>(value: T | null | undefined): value is T {
|
||||
return value !== null && value !== undefined;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { fetchTextWithUserAgent } from "../util/http";
|
||||
import { downloadBinary } from "./binary-http.ios";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { fromData, toData } from "./kotlin-data";
|
||||
import type { WebSignIn } from "./spotify-auth";
|
||||
import { SPOTIFY_IOS_BUNDLE_ID } from "./spotify-player";
|
||||
import { SpotifyService } from "./spotify-service";
|
||||
|
||||
declare const FaceclawAppleMusic: any;
|
||||
declare const FaceclawCrypto: any;
|
||||
declare const FaceclawGraphics: any;
|
||||
declare const FaceclawKitIosProtocol: any;
|
||||
declare const FaceclawWebSignIn: any;
|
||||
|
||||
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Sign-in in the system browser sheet, which FaceclawWebSignIn closes once the loopback redirect arrives. */
|
||||
const webSignIn: WebSignIn = {
|
||||
createPkce: () => {
|
||||
// 64 URL-safe characters (RFC 7636 allows 43..128).
|
||||
const verifier = String(FaceclawCrypto.randomBase64Url(48));
|
||||
return { verifier, challenge: String(FaceclawCrypto.sha256Base64Url(verifier)) };
|
||||
},
|
||||
randomState: () => String(FaceclawCrypto.randomBase64Url(16)),
|
||||
open: (authorizeUrl, port, path) =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
FaceclawWebSignIn.shared().signInPortPathCompletion(authorizeUrl, port, path, (redirectUrl: string | null, error: string | null) => {
|
||||
if (redirectUrl) resolve(String(redirectUrl));
|
||||
else reject(new Error(String(error || "Sign-in failed")));
|
||||
});
|
||||
}),
|
||||
cancel: () => FaceclawWebSignIn.shared().cancel(),
|
||||
};
|
||||
|
||||
let protocol: any = null;
|
||||
let service: SpotifyService | null = null;
|
||||
|
||||
/** The Spotify service, created on first use (main thread only: it owns timers and the sign-in sheet). */
|
||||
export function spotifyService(): SpotifyService | null {
|
||||
service ??= new SpotifyService({
|
||||
packageName: SPOTIFY_IOS_BUNDLE_ID,
|
||||
fetch: (url, init) => fetchTextWithUserAgent(url, init),
|
||||
webSignIn,
|
||||
download: async (url) => {
|
||||
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
|
||||
return response.status === 200 ? response.bytes : null;
|
||||
},
|
||||
decodeArt: (bytes, maxSize) => {
|
||||
const packet = FaceclawGraphics.bgraPacketFromImageDataMaxSize(toData(bytes), Math.round(maxSize));
|
||||
if (!packet) return null;
|
||||
// The shared Kotlin tone curve and dither, as for Apple Music's art.
|
||||
protocol ??= FaceclawKitIosProtocol.new();
|
||||
return grayImageFromPacket(fromData(protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither)));
|
||||
},
|
||||
// FaceclawAppleMusic's system output volume needs no media-library access.
|
||||
systemVolume: {
|
||||
get: () => {
|
||||
const volume = Number(FaceclawAppleMusic.shared().volumePercent());
|
||||
return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1;
|
||||
},
|
||||
set: (percent) => FaceclawAppleMusic.shared().setVolumePercent(Math.max(0, Math.min(100, Math.round(percent)))),
|
||||
},
|
||||
});
|
||||
return service;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { Application, Utils } from "@nativescript/core";
|
||||
import { fetchTextWithUserAgent } from "../util/http";
|
||||
import { downloadBinary } from "./binary-http";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { copyToJavaByteBuffer } from "./java-direct-buffer";
|
||||
import type { WebSignIn } from "./spotify-auth";
|
||||
import { SPOTIFY_ANDROID_PACKAGE } from "./spotify-player";
|
||||
import { SpotifyService } from "./spotify-service";
|
||||
|
||||
declare const android: any;
|
||||
declare const com: any;
|
||||
|
||||
const ART_DOWNLOAD_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** The sign-in's listener proxy, held while Java may still call it. */
|
||||
let signInListener: unknown = null;
|
||||
|
||||
/** Faceclaw's activity is in front, so it may open the browser (Android blocks background activity starts). */
|
||||
function foregroundActivity(): any {
|
||||
return Application.suspended ? null : Application.android?.foregroundActivity ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sign-in in the phone's browser, with FaceclawWebSignIn catching the
|
||||
* loopback redirect. A sign-in started from the glasses while Faceclaw is in
|
||||
* the background opens the page when Faceclaw next comes to the front, as on
|
||||
* iOS.
|
||||
*/
|
||||
const webSignIn: WebSignIn = {
|
||||
createPkce: () => {
|
||||
// 64 URL-safe characters (RFC 7636 allows 43..128).
|
||||
const verifier = String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(48));
|
||||
return { verifier, challenge: String(com.faceclaw.app.FaceclawWebSignIn.sha256Base64Url(verifier)) };
|
||||
},
|
||||
randomState: () => String(com.faceclaw.app.FaceclawWebSignIn.randomBase64Url(16)),
|
||||
open: (authorizeUrl, port, path) =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
const native = com.faceclaw.app.FaceclawWebSignIn.shared();
|
||||
let onResume: ((args: any) => void) | null = null;
|
||||
const stopWaiting = () => {
|
||||
if (onResume) Application.off(Application.resumeEvent, onResume);
|
||||
onResume = null;
|
||||
};
|
||||
const listener = new com.faceclaw.app.FaceclawWebSignInListener({
|
||||
onFinished: (redirectUrl: string | null, error: string | null) => {
|
||||
if (signInListener === listener) signInListener = null;
|
||||
stopWaiting();
|
||||
if (redirectUrl) resolve(String(redirectUrl));
|
||||
else reject(new Error(String(error || "Sign-in failed")));
|
||||
},
|
||||
});
|
||||
const startError = native.start(port, path, listener);
|
||||
if (startError) {
|
||||
reject(new Error(String(startError)));
|
||||
return;
|
||||
}
|
||||
signInListener = listener;
|
||||
const present = (activity: any) => {
|
||||
stopWaiting();
|
||||
const openError = native.openBrowser(activity, authorizeUrl);
|
||||
if (openError) native.fail(String(openError));
|
||||
};
|
||||
const activity = foregroundActivity();
|
||||
if (activity) {
|
||||
present(activity);
|
||||
return;
|
||||
}
|
||||
onResume = (args: any) => present(args?.activity ?? args?.android ?? Utils.android.getApplicationContext());
|
||||
Application.on(Application.resumeEvent, onResume);
|
||||
}),
|
||||
cancel: () => com.faceclaw.app.FaceclawWebSignIn.shared().cancel(),
|
||||
};
|
||||
|
||||
/** The phone's media-stream volume, for Spotify playing on this phone (as the session bridge reports it). */
|
||||
function audioManager(): any {
|
||||
return Utils.android.getApplicationContext()?.getSystemService(android.content.Context.AUDIO_SERVICE) ?? null;
|
||||
}
|
||||
|
||||
const systemVolume = {
|
||||
get: (): number => {
|
||||
const audio = audioManager();
|
||||
if (!audio) return -1;
|
||||
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
|
||||
if (max <= 0) return 0;
|
||||
return Math.max(0, Math.min(100, Math.round((Number(audio.getStreamVolume(android.media.AudioManager.STREAM_MUSIC)) * 100) / max)));
|
||||
},
|
||||
set: (percent: number): void => {
|
||||
const audio = audioManager();
|
||||
if (!audio) return;
|
||||
const max = Number(audio.getStreamMaxVolume(android.media.AudioManager.STREAM_MUSIC));
|
||||
const volume = Math.round((Math.max(0, Math.min(100, percent)) * max) / 100);
|
||||
audio.setStreamVolume(android.media.AudioManager.STREAM_MUSIC, volume, 0);
|
||||
},
|
||||
};
|
||||
|
||||
let service: SpotifyService | null = null;
|
||||
|
||||
/**
|
||||
* The Spotify Web API service, created on first use (main thread only: it
|
||||
* owns timers and the sign-in). Signing in is optional on Android: Spotify's
|
||||
* media session already gives now-playing and transport control, and the
|
||||
* media bridges add the Web API's extras once signed in (library browsing,
|
||||
* search, shuffle / repeat / like, playback on other Spotify devices).
|
||||
*/
|
||||
export function spotifyService(): SpotifyService | null {
|
||||
service ??= new SpotifyService({
|
||||
packageName: SPOTIFY_ANDROID_PACKAGE,
|
||||
fetch: (url, init) => fetchTextWithUserAgent(url, init),
|
||||
webSignIn,
|
||||
download: async (url) => {
|
||||
const response = await downloadBinary(url, ART_DOWNLOAD_TIMEOUT_MS);
|
||||
return response.status === 200 ? response.bytes : null;
|
||||
},
|
||||
decodeArt: (bytes, maxSize) => {
|
||||
const size = Math.round(maxSize);
|
||||
// The shared photo tone (gamma + dither), as for media-session art.
|
||||
return grayImageFromPacket(
|
||||
com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(copyToJavaByteBuffer(bytes), size, size, PHOTO_TONE.gamma, PHOTO_TONE.dither),
|
||||
);
|
||||
},
|
||||
systemVolume,
|
||||
});
|
||||
return service;
|
||||
}
|
||||
+159
-16
@@ -2,7 +2,9 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
export type IosMicrophoneSession = {
|
||||
@@ -10,20 +12,26 @@ export type IosMicrophoneSession = {
|
||||
}
|
||||
/** How long a cloud provider gets, after the last audio, to send its final transcript. */
|
||||
const CLOUD_FINAL_TIMEOUT_MS = 5000
|
||||
/** Retries of a failed raw-tap start (a new session's display path is usually still coming up). */
|
||||
const RAW_RETRY_MS = 3000
|
||||
const RAW_RETRY_LIMIT = 5
|
||||
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||
* the PCM capture runs open-ended, and pauses commit its segments. When the
|
||||
* session or the app lifecycle cuts one short it is drained, not discarded;
|
||||
* restarting it is the controller's call.
|
||||
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
|
||||
* is open-ended and has pauses commit its segments, so it needs a cloud
|
||||
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
|
||||
* recognizer only handles short dictations. When the session or the app
|
||||
* lifecycle cuts one short it is drained, not discarded; restarting it is the
|
||||
* controller's call.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
private readonly statuses = new Set<(state: VoiceControlState) => void>()
|
||||
private readonly transcripts = new Set<(event: VoiceTranscriptEvent) => void>()
|
||||
private readonly ends = new Set<() => void>()
|
||||
private readonly levels = new Set<(level: number) => void>()
|
||||
private status = 'Voice input ready.'
|
||||
private generation = 0
|
||||
private session: IosMicrophoneSession | null = null
|
||||
@@ -35,10 +43,13 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
|
||||
// on-device continuous capture) owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: ReconnectingSttClient | null = null
|
||||
// What transcribes the current capture, for the log.
|
||||
private recognition = 'on-device'
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||
@@ -51,6 +62,21 @@ export class IosVoiceControlBridge {
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||
// rawListeners while any owner wants it, from the session setRawSession
|
||||
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||
// it; it resumes on its own once that capture has ended.
|
||||
private rawNative: any = null
|
||||
private readonly rawListeners = new Set<(pcm: Uint8Array) => void>()
|
||||
private readonly rawOwners = new Set<string>()
|
||||
private rawSession: IosMicrophoneSession | null = null
|
||||
// The session the tap enabled the mic on, while it is starting or live.
|
||||
private rawHeld: IosMicrophoneSession | null = null
|
||||
private rawLive = false
|
||||
private rawGeneration = 0
|
||||
private rawFailures = 0
|
||||
private rawRetryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private rawLog: (message: string) => void = () => {}
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
|
||||
@@ -60,15 +86,19 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
onSpeechEnd(listener: () => void): () => void { this.ends.add(listener); return () => { this.ends.delete(listener) } }
|
||||
onSpeechPause(_listener: () => void): () => void { return () => {} }
|
||||
/** The mic level (see pcmLevel) of each PCM chunk; only cloud captures and the raw tap supply PCM. */
|
||||
onMicLevel(listener: (level: number) => void): () => void { this.levels.add(listener); return () => { this.levels.delete(listener) } }
|
||||
private setStatus(status: string): void { this.status = status; for (const fn of [...this.statuses]) fn({ status, listening: this.capturing, detail: '' }) }
|
||||
private ensureNative(): any {
|
||||
if (!this.native) {
|
||||
this.native = FaceclawSpeech.new()
|
||||
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
|
||||
this.native.pcmHandler = (pcm: NSData) => {
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
||||
// Copied: bufferFromData doesn't retain pcm, and STT clients queue chunks.
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||
this.cloud?.acceptPcm(bytes)
|
||||
if (this.continuous) this.record(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.native
|
||||
@@ -96,6 +126,8 @@ export class IosVoiceControlBridge {
|
||||
get statusText(): string { return this.status }
|
||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
// Speech capture preempts the raw tap; its mic release is queued before this enable, which wins.
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
const generation = ++this.generation
|
||||
this.continuous = continuous
|
||||
@@ -111,7 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -125,24 +157,34 @@ export class IosVoiceControlBridge {
|
||||
}
|
||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||
if (this.joinsContinuous(continuous)) return
|
||||
this.endRawTap()
|
||||
this.stop()
|
||||
this.continuous = continuous
|
||||
this.log = log; this.source = 'phone'; this.capturing = true
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => this.continuous)
|
||||
const options = {
|
||||
language: voice.language,
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: (status: string) => { if (current()) this.setStatus(status) },
|
||||
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}
|
||||
const continuous = () => this.continuous
|
||||
let cloud = createCloudSttClient(voice, options, continuous)
|
||||
this.recognition = cloud ? 'cloud' : 'on-device'
|
||||
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
|
||||
if (!cloud && this.continuous && speechAnalyzerSupported()) {
|
||||
cloud = createSpeechAnalyzerClient(options, continuous)
|
||||
this.recognition = 'on-device (SpeechAnalyzer)'
|
||||
}
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
@@ -155,8 +197,11 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
|
||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||
}
|
||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||
/**
|
||||
* Whether continuous capture can run with this provider: a cloud one, or
|
||||
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
|
||||
*/
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
|
||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
@@ -222,6 +267,100 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
||||
this.stop(); this.setStatus(message); this.emitEnd()
|
||||
}
|
||||
/** Subscribe to the raw tap's decoded glasses PCM (16 kHz mono S16LE). */
|
||||
onRawPcm(listener: (pcm: Uint8Array) => void): () => void {
|
||||
this.rawListeners.add(listener); return () => { this.rawListeners.delete(listener) }
|
||||
}
|
||||
/**
|
||||
* The glasses session the raw tap captures from (connected and unlocked),
|
||||
* or null. Owners outlive a change: the tap moves to the next session.
|
||||
*/
|
||||
setRawSession(session: IosMicrophoneSession | null, log: (message: string) => void): void {
|
||||
this.rawLog = log
|
||||
if (session === this.rawSession) return
|
||||
this.endRawTap()
|
||||
this.rawSession = session; this.rawFailures = 0
|
||||
this.syncRawTap()
|
||||
}
|
||||
/**
|
||||
* Start the decode-only raw tap on behalf of `owner`; the communicator is
|
||||
* the one setRawSession bound, not `options.communicator`. False while
|
||||
* speech capture holds the mic or no session is bound: the owner stays
|
||||
* registered and the tap starts once it can. Idempotent while running.
|
||||
*/
|
||||
startRawCapture(options: { communicator: any; owner: string }): boolean {
|
||||
if (!this.rawOwners.has(options.owner)) { this.rawOwners.add(options.owner); this.rawFailures = 0 }
|
||||
this.syncRawTap()
|
||||
return this.rawHeld !== null
|
||||
}
|
||||
/** `owner` no longer wants the raw tap; it stops once no owner does. */
|
||||
stopRawCapture(owner: string): void {
|
||||
if (this.rawOwners.delete(owner)) this.syncRawTap()
|
||||
}
|
||||
/** Whether the raw tap's glasses mic enable went through and still holds. */
|
||||
isRawCaptureLive(): boolean { return this.rawLive }
|
||||
/** Run the tap while an owner wants it, a session is bound, and no speech capture holds the mic. */
|
||||
private syncRawTap(): void {
|
||||
const session = this.rawSession
|
||||
if (!session || !this.rawOwners.size || this.capturing || this.finalizing) { this.endRawTap(); return }
|
||||
if (this.rawHeld || this.rawRetryTimer !== null) return
|
||||
void this.beginRawTap(session)
|
||||
}
|
||||
private async beginRawTap(session: IosMicrophoneSession): Promise<void> {
|
||||
const generation = ++this.rawGeneration
|
||||
const native = this.ensureRawNative()
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(false, false) ?? '')
|
||||
if (error) { this.failRawTap(error); return }
|
||||
this.rawHeld = session
|
||||
try {
|
||||
await session.setMicrophone(true, packet => {
|
||||
if (generation !== this.rawGeneration) return
|
||||
const copy = new Uint8Array(packet)
|
||||
native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.rawGeneration) return
|
||||
this.rawLive = true
|
||||
this.rawLog('EvenHub mic: glasses microphone enabled')
|
||||
} catch (failure) {
|
||||
if (generation === this.rawGeneration) this.failRawTap(failure instanceof Error ? failure.message : String(failure))
|
||||
}
|
||||
}
|
||||
private ensureRawNative(): any {
|
||||
if (!this.rawNative) {
|
||||
this.rawNative = FaceclawSpeech.new()
|
||||
// A PCM capture only ends on its own when decoding failed or fell behind.
|
||||
this.rawNative.eventHandler = (json: string) => {
|
||||
const event = JSON.parse(json)
|
||||
if (event.kind === 'ended' && this.rawHeld) this.failRawTap(String(event.message ?? 'audio ended'))
|
||||
}
|
||||
this.rawNative.pcmHandler = (pcm: NSData) => {
|
||||
// Copied: bufferFromData doesn't retain pcm.
|
||||
const bytes = new Uint8Array(interop.bufferFromData(pcm)).slice()
|
||||
// Audio is flowing, so a later failure is a new problem worth retrying.
|
||||
this.rawFailures = 0
|
||||
for (const fn of [...this.rawListeners]) fn(bytes)
|
||||
if (this.levels.size) { const level = pcmLevel(bytes); for (const fn of [...this.levels]) fn(level) }
|
||||
}
|
||||
}
|
||||
return this.rawNative
|
||||
}
|
||||
/** Drop the tap and retry it shortly, a bounded number of times. */
|
||||
private failRawTap(message: string): void {
|
||||
this.rawLog(`EvenHub mic: ${message}`)
|
||||
this.endRawTap()
|
||||
if (++this.rawFailures >= RAW_RETRY_LIMIT) return
|
||||
this.rawRetryTimer = setTimeout(() => { this.rawRetryTimer = null; this.syncRawTap() }, RAW_RETRY_MS)
|
||||
}
|
||||
/** Stop the tap; owners stay. Releases the mic only if the tap enabled it. */
|
||||
private endRawTap(): void {
|
||||
++this.rawGeneration
|
||||
if (this.rawRetryTimer !== null) clearTimeout(this.rawRetryTimer)
|
||||
this.rawRetryTimer = null
|
||||
this.rawLive = false
|
||||
this.rawNative?.cancel()
|
||||
const session = this.rawHeld; this.rawHeld = null
|
||||
if (session) void session.setMicrophone(false).catch(error => this.rawLog(`EvenHub mic cleanup: ${error}`))
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
const session = this.session; this.session = null
|
||||
@@ -255,7 +394,11 @@ export class IosVoiceControlBridge {
|
||||
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
|
||||
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
|
||||
}
|
||||
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
|
||||
private emitEnd(): void {
|
||||
for (const fn of [...this.ends]) fn()
|
||||
// After the listeners: one may start the next capture, which keeps the mic.
|
||||
this.syncRawTap()
|
||||
}
|
||||
private finishCompletion(): void {
|
||||
const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -56,8 +57,9 @@ export type PushToTalkOptions = {
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||
@@ -138,6 +140,7 @@ export class FaceclawVoiceControlBridge {
|
||||
// stopped for its duration.
|
||||
private readonly rawPcmListeners = new Set<RawPcmListener>();
|
||||
private readonly frameMetaListeners = new Set<FrameMetaListener>();
|
||||
private readonly micLevelListeners = new Set<(level: number) => void>();
|
||||
private rawActive = false;
|
||||
// Who wants the raw tap. It runs while any owner does, so one owner letting
|
||||
// go (an EvenHub app backgrounding) doesn't silence another.
|
||||
@@ -237,6 +240,16 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.rawPcmListeners.delete(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to the mic level (meter position in [0,1], see pcmLevel) of
|
||||
* each decoded audio chunk, about every 50 ms, from any capture: STT in
|
||||
* every mode, or the raw tap.
|
||||
*/
|
||||
onMicLevel(listener: (level: number) => void): () => void {
|
||||
this.micLevelListeners.add(listener);
|
||||
return () => this.micLevelListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
|
||||
onFrameMeta(listener: FrameMetaListener): () => void {
|
||||
this.frameMetaListeners.add(listener);
|
||||
@@ -539,6 +552,10 @@ export class FaceclawVoiceControlBridge {
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
if (this.micLevelListeners.size > 0) {
|
||||
const level = pcmLevel(bytes);
|
||||
this.micLevelListeners.forEach((listener) => listener(level));
|
||||
}
|
||||
},
|
||||
onFrameMeta: (angleDegrees: number, ssr: number) => {
|
||||
if (this.frameMetaListeners.size === 0) return;
|
||||
|
||||
@@ -1,18 +1,40 @@
|
||||
<GridLayout rows="auto,*,auto" maxHeight="280" xmlns="http://schemas.nativescript.org/tns.xsd"
|
||||
class="system-name-modal"
|
||||
class="system-name-modal keyboard-input-panel"
|
||||
visibility="{{ keyboardInputVisibility }}"
|
||||
horizontalAlignment="stretch"
|
||||
verticalAlignment="bottom">
|
||||
<Label row="0" text="Keyboard input" class="h2" />
|
||||
<!-- Title, then the history button (recent sent messages) and the
|
||||
visibility button, which cycles both / phone only / glasses only and
|
||||
dims the half of its phone+glasses pair where the dialog is hidden.
|
||||
rows="auto" on the grids: an implicit star row balloons in here. -->
|
||||
<GridLayout row="0" rows="auto" columns="*,auto,auto">
|
||||
<Label col="0" text="Keyboard input" class="h2" verticalAlignment="center" />
|
||||
<GridLayout col="1" rows="auto" class="panel-icon-button" opacity="{{ keyboardInputHistoryOpacity }}"
|
||||
accessible="true" accessibilityRole="button" accessibilityLabel="Recent inputs"
|
||||
tap="{{ onKeyboardInputHistoryTap }}">
|
||||
<Image src="res://ic_input_history" class="panel-icon" />
|
||||
</GridLayout>
|
||||
<StackLayout col="2" orientation="horizontal" class="panel-icon-button"
|
||||
accessible="true" accessibilityRole="button" accessibilityLabel="{{ keyboardInputVisibilityDescription }}"
|
||||
tap="{{ onKeyboardInputVisibilityTap }}">
|
||||
<Image src="res://ic_show_phone" class="panel-icon" opacity="{{ keyboardInputPhoneIconOpacity }}" />
|
||||
<Image src="res://ic_show_glasses" class="panel-icon" opacity="{{ keyboardInputGlassesIconOpacity }}" />
|
||||
</StackLayout>
|
||||
</GridLayout>
|
||||
<TextView row="1"
|
||||
id="keyboardInputField"
|
||||
class="{{ keyboardInputFieldClass }}"
|
||||
text="{{ keyboardInputText }}"
|
||||
hint="Type a message"
|
||||
textChange="{{ onKeyboardInputTextChange }}" />
|
||||
<!-- Wraps onto a second line when the labels don't fit side by side. -->
|
||||
<FlexboxLayout row="2" flexWrap="wrap" class="m-t-8">
|
||||
<Label row="1" text="Shown on the glasses only. Tap to type." textWrap="true"
|
||||
class="keyboard-input-hidden-hint"
|
||||
visibility="{{ keyboardInputGlassesOnlyVisibility }}"
|
||||
tap="{{ onKeyboardInputGlassesOnlyHintTap }}" />
|
||||
<!-- One strip: at most two destinations (assistant, app) plus Discard. -->
|
||||
<FlexboxLayout row="2" flexWrap="nowrap" class="keyboard-input-buttons">
|
||||
<Button text="{{ keyboardInputPrimaryTargetLabel }}" tap="{{ onKeyboardInputPrimarySendTap }}" visibility="{{ keyboardInputPrimaryTargetVisibility }}" flexGrow="1" class="-primary" />
|
||||
<Button text="{{ keyboardInputSecondaryTargetLabel }}" tap="{{ onKeyboardInputSecondarySendTap }}" visibility="{{ keyboardInputSecondaryTargetVisibility }}" flexGrow="1" />
|
||||
<Button text="Discard" tap="{{ onKeyboardInputDiscardTap }}" flexGrow="1" />
|
||||
<Button text="{{ keyboardInputSecondaryTargetLabel }}" tap="{{ onKeyboardInputSecondarySendTap }}" visibility="{{ keyboardInputSecondaryTargetVisibility }}" flexGrow="1" class="keyboard-input-gap" />
|
||||
<Button text="Discard" tap="{{ onKeyboardInputDiscardTap }}" flexGrow="1" class="keyboard-input-gap" />
|
||||
</FlexboxLayout>
|
||||
</GridLayout>
|
||||
@@ -1,7 +1,32 @@
|
||||
import { Observable } from '@nativescript/core'
|
||||
import type { KeyboardInputSession } from '../ui/shell/keyboard-input'
|
||||
import { Dialogs, Observable, type EventData, type TextView, type View } from '@nativescript/core'
|
||||
import type { KeyboardInputSession, KeyboardInputVisibility } from '../ui/shell/keyboard-input'
|
||||
|
||||
/** Phone editor for the shell's keyboard session, using the shared XML panel. */
|
||||
/** The visibility button's cycle: both → phone only → glasses only → both. */
|
||||
const NEXT_VISIBILITY: Record<KeyboardInputVisibility, KeyboardInputVisibility> = { both: 'phone', phone: 'glasses', glasses: 'both' }
|
||||
const VISIBILITY_DESCRIPTIONS: Record<KeyboardInputVisibility, string> = {
|
||||
both: 'Shown on phone and glasses', phone: 'Shown on phone only', glasses: 'Shown on glasses only',
|
||||
}
|
||||
/**
|
||||
* Phone button text for the glasses' send-target labels, short enough for
|
||||
* every button to share one strip. Unlisted labels (the assistant overlay's
|
||||
* "Send") are already short.
|
||||
*/
|
||||
const SHORT_TARGET_LABELS: Record<string, string> = { 'Send to Assistant': 'To Assistant', 'Type Into App': 'To App' }
|
||||
const HISTORY_LABEL_LENGTH = 80
|
||||
const DIMMED_ICON_OPACITY = 0.3
|
||||
|
||||
/** One history entry as a single line, truncated for the picker. */
|
||||
function historyLabel(text: string): string {
|
||||
const line = text.replace(/\s+/g, ' ').trim()
|
||||
return line.length > HISTORY_LABEL_LENGTH ? `${line.slice(0, HISTORY_LABEL_LENGTH - 1)}…` : line
|
||||
}
|
||||
|
||||
/**
|
||||
* Phone editor for the shell's keyboard session: the binding context of the
|
||||
* shared keyboard-input-panel.xml on both platforms (Android's MainViewModel
|
||||
* hands it the session from the dashboard controller, iOS's main page from
|
||||
* the preview controller).
|
||||
*/
|
||||
export class KeyboardInputViewModel extends Observable {
|
||||
private session: KeyboardInputSession | null = null
|
||||
private text = ''
|
||||
@@ -10,17 +35,24 @@ export class KeyboardInputViewModel extends Observable {
|
||||
if (session === this.session) return
|
||||
this.session = session
|
||||
this.text = ''
|
||||
for (const property of ['keyboardInputText', 'keyboardInputVisibility', 'keyboardInputPrimaryTargetLabel',
|
||||
'keyboardInputPrimaryTargetVisibility', 'keyboardInputSecondaryTargetLabel', 'keyboardInputSecondaryTargetVisibility']) {
|
||||
this.notifyPropertyChange(property, this.get(property))
|
||||
}
|
||||
this.notifyPanel(['keyboardInputText', 'keyboardInputVisibility', 'keyboardInputPrimaryTargetLabel',
|
||||
'keyboardInputPrimaryTargetVisibility', 'keyboardInputSecondaryTargetLabel', 'keyboardInputSecondaryTargetVisibility',
|
||||
'keyboardInputHistoryOpacity'])
|
||||
this.notifyVisibilityMode()
|
||||
}
|
||||
private notifyPanel(properties: string[]): void {
|
||||
for (const property of properties) this.notifyPropertyChange(property, this.get(property))
|
||||
}
|
||||
private notifyVisibilityMode(): void {
|
||||
this.notifyPanel(['keyboardInputPhoneIconOpacity', 'keyboardInputGlassesIconOpacity', 'keyboardInputVisibilityDescription',
|
||||
'keyboardInputFieldClass', 'keyboardInputGlassesOnlyVisibility'])
|
||||
}
|
||||
get keyboardInputVisibility(): 'visible' | 'collapse' { return this.session ? 'visible' : 'collapse' }
|
||||
get keyboardInputText(): string { return this.text }
|
||||
set keyboardInputText(text: string) { this.text = text; this.session?.setText(text) }
|
||||
private targetLabel(index: number): string {
|
||||
const target = this.session?.targets[index]
|
||||
return target?.id === 'assistant' ? 'Send to Agent' : target?.id === 'app' ? 'Type into App' : target?.label ?? ''
|
||||
const label = this.session?.targets[index]?.label ?? ''
|
||||
return SHORT_TARGET_LABELS[label] ?? label
|
||||
}
|
||||
get keyboardInputPrimaryTargetLabel(): string { return this.targetLabel(0) }
|
||||
get keyboardInputSecondaryTargetLabel(): string { return this.targetLabel(1) }
|
||||
@@ -38,4 +70,61 @@ export class KeyboardInputViewModel extends Observable {
|
||||
this.session.sendTo(target.id)
|
||||
}
|
||||
onKeyboardInputDiscardTap(): void { this.session?.discard() }
|
||||
|
||||
// ---- visibility button: where the dialog shows ----
|
||||
//
|
||||
// Glasses-only keeps the text field focused (so the IME keeps typing into
|
||||
// it) but shrinks it to an invisible sliver; a hint line stands in for it
|
||||
// and refocuses it if the IME was dismissed.
|
||||
|
||||
private get visibilityMode(): KeyboardInputVisibility { return this.session?.visibility ?? 'both' }
|
||||
get keyboardInputPhoneIconOpacity(): number { return this.visibilityMode === 'glasses' ? DIMMED_ICON_OPACITY : 1 }
|
||||
get keyboardInputGlassesIconOpacity(): number { return this.visibilityMode === 'phone' ? DIMMED_ICON_OPACITY : 1 }
|
||||
get keyboardInputVisibilityDescription(): string { return VISIBILITY_DESCRIPTIONS[this.visibilityMode] }
|
||||
get keyboardInputFieldClass(): string { return this.visibilityMode === 'glasses' ? 'keyboard-input-field-hidden' : '' }
|
||||
get keyboardInputGlassesOnlyVisibility(): 'visible' | 'collapse' { return this.visibilityMode === 'glasses' ? 'visible' : 'collapse' }
|
||||
onKeyboardInputVisibilityTap(): void {
|
||||
if (!this.session) return
|
||||
this.session.setVisibility(NEXT_VISIBILITY[this.visibilityMode])
|
||||
this.notifyVisibilityMode()
|
||||
}
|
||||
onKeyboardInputGlassesOnlyHintTap(args: EventData): void {
|
||||
focusField(args.object as View)
|
||||
}
|
||||
|
||||
// ---- history button: recent sent messages (in memory, newest first) ----
|
||||
|
||||
get keyboardInputHistoryOpacity(): number { return this.session?.history.length ? 1 : DIMMED_ICON_OPACITY }
|
||||
async onKeyboardInputHistoryTap(args: EventData): Promise<void> {
|
||||
const session = this.session
|
||||
if (!session?.history.length) return
|
||||
const entries = new Map<string, string>()
|
||||
for (const entry of session.history) {
|
||||
const label = historyLabel(entry)
|
||||
if (!entries.has(label)) entries.set(label, entry)
|
||||
}
|
||||
const choice = await Dialogs.action({ title: 'Recent inputs', cancelButtonText: 'Cancel', actions: [...entries.keys()] })
|
||||
const text = entries.get(choice)
|
||||
if (text === undefined || session !== this.session) return
|
||||
const field = fieldNear(args.object as View)
|
||||
this.keyboardInputText = text
|
||||
// Through the view, not a property change: its textChange handler then
|
||||
// syncs the model and session the same way typing does.
|
||||
if (field) field.text = text
|
||||
focusField(args.object as View, text.length)
|
||||
}
|
||||
}
|
||||
|
||||
function fieldNear(view: View): TextView | undefined {
|
||||
return view.page?.getViewById<TextView>('keyboardInputField')
|
||||
}
|
||||
|
||||
/** Focus the panel's field (bringing the IME back), optionally placing the caret. */
|
||||
function focusField(view: View, caret?: number): void {
|
||||
setTimeout(() => {
|
||||
const field = fieldNear(view)
|
||||
if (!field) return
|
||||
field.focus()
|
||||
if (caret !== undefined) field.setSelection(caret)
|
||||
}, 0)
|
||||
}
|
||||
@@ -182,7 +182,8 @@
|
||||
the phone's end of the keyboard dialog on the glasses
|
||||
(ui/shell/keyboard-input.ts). Same placement and collapsing rules
|
||||
as the setting editor above. Enter inserts a newline; the send
|
||||
buttons pick a destination directly. -->
|
||||
<rc:keyboard-input-panel />
|
||||
buttons pick a destination directly. Bound to its own model
|
||||
(KeyboardInputViewModel, shared with iOS). -->
|
||||
<rc:keyboard-input-panel bindingContext="{{ keyboardInputModel }}" />
|
||||
</GridLayout>
|
||||
</Page>
|
||||
@@ -1,4 +1,5 @@
|
||||
import { RemoteControlsViewModel } from './remote-controls-view-model';
|
||||
import { KeyboardInputViewModel } from './keyboard-input-view-model';
|
||||
import { BleBandwidthMeter } from "./ble-bandwidth-meter";
|
||||
import {
|
||||
Application,
|
||||
@@ -52,8 +53,6 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
private _activeTextEditorToggleValue = false;
|
||||
private _activeTextEditorToggleVisible = false;
|
||||
private _keyboardInputActive = false;
|
||||
private _keyboardInputTargets: ReadonlyArray<{ id: string; label: string }> = [];
|
||||
private _keyboardInputText = "";
|
||||
private _evenAppConflictMessage = "";
|
||||
private _evenAppConflictWarningVisible = false;
|
||||
private _firmwareWarningMessage = "";
|
||||
@@ -103,8 +102,8 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
this.activeTextEditorToggleLabel = snapshot.activeTextEditorToggleLabel;
|
||||
this.activeTextEditorToggleValue = snapshot.activeTextEditorToggleValue;
|
||||
this.activeTextEditorToggleVisible = snapshot.activeTextEditorToggleVisible;
|
||||
// Targets before active: the panel's buttons are labelled when it appears.
|
||||
this.keyboardInputTargets = snapshot.keyboardInputTargets;
|
||||
// Session before active: the panel is filled in when it appears.
|
||||
this.keyboardInputModel.setSession(dashboardController.keyboardInputSession);
|
||||
this.keyboardInputActive = snapshot.keyboardInputActive;
|
||||
this.evenAppConflictMessage = snapshot.evenAppConflictMessage;
|
||||
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
|
||||
@@ -494,11 +493,13 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
|
||||
// ---- keyboard-input panel (the keyboard button beside the mic button) ----
|
||||
//
|
||||
// The dialog lives on the glasses (ui/shell/keyboard-input.ts); this panel
|
||||
// is the phone's end of it: the text field the IME types into and a send
|
||||
// button per destination. The field's text is the panel's own state, never
|
||||
// echoed back from the controller (see setActiveTextSettingValue for why);
|
||||
// it is cleared whenever a dialog opens or closes.
|
||||
// The dialog lives on the glasses (ui/shell/keyboard-input.ts); the panel
|
||||
// is the phone's end of it, bound to keyboardInputModel (shared with iOS).
|
||||
// Its text is the panel's own state, never echoed back from the controller
|
||||
// (see setActiveTextSettingValue for why); it is cleared whenever a dialog
|
||||
// opens or closes.
|
||||
|
||||
readonly keyboardInputModel = new KeyboardInputViewModel();
|
||||
|
||||
get keyboardInputActive(): boolean {
|
||||
return this._keyboardInputActive;
|
||||
@@ -507,78 +508,14 @@ export class MainViewModel extends RemoteControlsViewModel {
|
||||
set keyboardInputActive(value: boolean) {
|
||||
if (this._keyboardInputActive === value) return;
|
||||
this._keyboardInputActive = value;
|
||||
this.keyboardInputText = "";
|
||||
this.notifyPropertyChange("keyboardInputActive", value);
|
||||
this.notifyPropertyChange("keyboardInputVisibility", this.keyboardInputVisibility);
|
||||
this.notifyPropertyChange("controlsVisibility", this.controlsVisibility);
|
||||
}
|
||||
|
||||
get keyboardInputVisibility(): "visible" | "collapse" {
|
||||
return this._keyboardInputActive ? "visible" : "collapse";
|
||||
}
|
||||
|
||||
get keyboardInputText(): string {
|
||||
return this._keyboardInputText;
|
||||
}
|
||||
|
||||
set keyboardInputText(value: string) {
|
||||
if (this._keyboardInputText === value) return;
|
||||
this._keyboardInputText = value;
|
||||
this.notifyPropertyChange("keyboardInputText", value);
|
||||
}
|
||||
|
||||
set keyboardInputTargets(value: ReadonlyArray<{ id: string; label: string }>) {
|
||||
this._keyboardInputTargets = value;
|
||||
this.notifyPropertyChange("keyboardInputPrimaryTargetLabel", this.keyboardInputPrimaryTargetLabel);
|
||||
this.notifyPropertyChange("keyboardInputPrimaryTargetVisibility", this.keyboardInputPrimaryTargetVisibility);
|
||||
this.notifyPropertyChange("keyboardInputSecondaryTargetLabel", this.keyboardInputSecondaryTargetLabel);
|
||||
this.notifyPropertyChange("keyboardInputSecondaryTargetVisibility", this.keyboardInputSecondaryTargetVisibility);
|
||||
}
|
||||
|
||||
// The glasses menu has at most two destinations (assistant, app), so the
|
||||
// panel has a fixed pair of buttons rather than a repeater.
|
||||
get keyboardInputPrimaryTargetLabel(): string {
|
||||
return this._keyboardInputTargets[0]?.label ?? "";
|
||||
}
|
||||
|
||||
get keyboardInputPrimaryTargetVisibility(): "visible" | "collapse" {
|
||||
return this._keyboardInputTargets[0] ? "visible" : "collapse";
|
||||
}
|
||||
|
||||
get keyboardInputSecondaryTargetLabel(): string {
|
||||
return this._keyboardInputTargets[1]?.label ?? "";
|
||||
}
|
||||
|
||||
get keyboardInputSecondaryTargetVisibility(): "visible" | "collapse" {
|
||||
return this._keyboardInputTargets[1] ? "visible" : "collapse";
|
||||
}
|
||||
|
||||
onKeyboardTap(): void {
|
||||
dashboardController.startKeyboardInput();
|
||||
}
|
||||
|
||||
onKeyboardInputTextChange(args: { value?: string; object?: { text?: string } }): void {
|
||||
const text = args.object?.text ?? args.value ?? "";
|
||||
// Track the draft so closing the dialog emits a clear even when the
|
||||
// binding hasn't updated the model. Avoid echoing edits into the IME.
|
||||
this._keyboardInputText = text;
|
||||
dashboardController.setKeyboardInputText(text);
|
||||
}
|
||||
|
||||
onKeyboardInputPrimarySendTap(): void {
|
||||
const target = this._keyboardInputTargets[0];
|
||||
if (target) dashboardController.sendKeyboardInput(target.id);
|
||||
}
|
||||
|
||||
onKeyboardInputSecondarySendTap(): void {
|
||||
const target = this._keyboardInputTargets[1];
|
||||
if (target) dashboardController.sendKeyboardInput(target.id);
|
||||
}
|
||||
|
||||
onKeyboardInputDiscardTap(): void {
|
||||
dashboardController.discardKeyboardInput();
|
||||
}
|
||||
|
||||
get evenAppConflictMessage(): string {
|
||||
return this._evenAppConflictMessage;
|
||||
}
|
||||
|
||||
+7
-2
@@ -85,15 +85,20 @@ export const TRANSLATE_SYSTEM_PROMPT =
|
||||
"Reply with the English translation only, with no preamble, quotes, notes, or transliteration. " +
|
||||
"If the speech is empty or meaningless noise, reply with nothing.";
|
||||
|
||||
/** User message for the Translate app: earlier turns for context, then the speech to translate. */
|
||||
/**
|
||||
* User message for the Translate app: earlier turns for context, then the
|
||||
* speech to translate. `language` names the spoken language when the user set
|
||||
* one (the transcription was told to expect it), else "".
|
||||
*/
|
||||
export function buildTranslateUserMessage(
|
||||
speech: string,
|
||||
context: ReadonlyArray<{ source: string; translation: string }>,
|
||||
language = "",
|
||||
): string {
|
||||
const earlier = context.length
|
||||
? "Earlier in this conversation (already translated; context only):\n" +
|
||||
context.map((turn) => `<speech>${turn.source}</speech>\n<english>${turn.translation}</english>`).join("\n") +
|
||||
"\n\n"
|
||||
: "";
|
||||
return `${earlier}Translate this speech into English:\n<speech>${speech}</speech>`;
|
||||
return `${earlier}Translate this ${language ? `${language} ` : ""}speech into English:\n<speech>${speech}</speech>`;
|
||||
}
|
||||
@@ -16,8 +16,10 @@ import {
|
||||
assistantModelProvider,
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed";
|
||||
import { MENU_ANIMATION_KEY } from "./menu-animation-pref";
|
||||
@@ -36,6 +38,7 @@ export type BatteryDisplayMode = "icon" | "percentage" | "stacked" | "stacked-pe
|
||||
export type BatteryIndicatorVisibility = "always" | "low" | "never";
|
||||
export type TimeFormat = "24h" | "12h";
|
||||
export type TemperatureUnit = "fahrenheit" | "celsius";
|
||||
export type DistanceUnit = "miles" | "kilometers";
|
||||
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "3m" | "never";
|
||||
// Auto runs Faceclaw's light curve. Numeric values use the calibrated firmware
|
||||
// scale, clamped to 2–100; 2 is also the endpoint for screen fades.
|
||||
@@ -321,14 +324,17 @@ export function batteryIndicatorSettingsKey(): string {
|
||||
].join("|");
|
||||
}
|
||||
|
||||
/** Exported so the Navigate worker can repaint when it changes. */
|
||||
export const TIME_FORMAT_SETTING_KEY = "display.timeFormat";
|
||||
|
||||
export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
|
||||
id: "timeFormat",
|
||||
label: "Time format",
|
||||
storageKey: "display.timeFormat",
|
||||
storageKey: TIME_FORMAT_SETTING_KEY,
|
||||
defaultValue: "24h",
|
||||
values: ["24h", "12h"],
|
||||
formatValue: timeFormatLabel,
|
||||
description: "Whether the top-bar clock shows 24-hour or 12-hour time.",
|
||||
description: "Whether the top-bar clock and other times of day, such as Navigate's arrival time, show 24-hour or 12-hour time.",
|
||||
});
|
||||
|
||||
export const temperatureUnitSetting = new ConfigSettingEnum<TemperatureUnit>({
|
||||
@@ -341,6 +347,19 @@ export const temperatureUnitSetting = new ConfigSettingEnum<TemperatureUnit>({
|
||||
description: "Whether the Weather app, its top-bar indicator, and the Glanceboard's system card show temperatures in Fahrenheit or Celsius.",
|
||||
});
|
||||
|
||||
/** Exported so the Navigate worker can repaint when it changes. */
|
||||
export const DISTANCE_UNIT_SETTING_KEY = "display.distanceUnit";
|
||||
|
||||
export const distanceUnitSetting = new ConfigSettingEnum<DistanceUnit>({
|
||||
id: "distanceUnit",
|
||||
label: "Distance unit",
|
||||
storageKey: DISTANCE_UNIT_SETTING_KEY,
|
||||
defaultValue: "miles",
|
||||
values: ["miles", "kilometers"],
|
||||
formatValue: (value) => value === "kilometers" ? "Kilometers" : "Miles",
|
||||
description: "Whether the Weather app shows wind speed in mph or km/h, and the Navigate app shows distances in miles and feet or kilometers and meters.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Where the app switcher (the open-window icons) sits: a strip along the
|
||||
* left or right edge of the display, a row under the window, or a popup box
|
||||
@@ -919,7 +938,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: global.isIOS ? "Cloud API" : "On-phone",
|
||||
direct: "On-phone",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
@@ -931,7 +950,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
||||
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys) or your configured OpenAI-compatible API, or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantHermesUrlSetting = new ConfigSettingString({
|
||||
@@ -997,6 +1016,48 @@ export const assistantAllowProactiveSetting = new ConfigSettingBoolean({
|
||||
"Let the external agent use glasses tools outside of a conversation, e.g. showing an alert when a long-running job finishes. Rate-limited; only tools marked proactive-safe are allowed.",
|
||||
});
|
||||
|
||||
export const assistantOpenAiCompatibleUrlSetting = new ConfigSettingString({
|
||||
id: "assistant-openai-compatible-url",
|
||||
label: "Compatible API URL",
|
||||
storageKey: "assistant.openAiCompatibleUrl",
|
||||
defaultValue: "",
|
||||
editorTitle: "OpenAI-compatible API root",
|
||||
glassesEditTitle: "Edit compatible API URL",
|
||||
formatValue: (value) => value || "(not set)",
|
||||
validate: (value) => (!value.trim() || normalizeOpenAiCompatibleUrl(value) ? null : "Enter an http:// or https:// API root."),
|
||||
description: "Explicit API root for the OpenAI-compatible Chat Completions assistant. Venice: https://api.venice.ai/api/v1. Open WebUI: https://your-host/api. The provider's path is not guessed.",
|
||||
});
|
||||
|
||||
export const assistantOpenAiCompatibleKeySetting = new ConfigSettingString({
|
||||
id: "assistant-openai-compatible-key",
|
||||
label: "Compatible API key",
|
||||
storageKey: "assistant.openAiCompatibleKey",
|
||||
defaultValue: "",
|
||||
editorTitle: "OpenAI-compatible API key (optional)",
|
||||
glassesEditTitle: "Edit compatible API key",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"),
|
||||
description: "Bearer key sent only to the compatible assistant URL above. Optional for private servers that do not require authentication.",
|
||||
});
|
||||
|
||||
export const assistantOpenAiCompatibleModelSetting = new ConfigSettingString({
|
||||
id: "assistant-openai-compatible-model",
|
||||
label: "Compatible model ID",
|
||||
storageKey: "assistant.openAiCompatibleModel",
|
||||
defaultValue: "",
|
||||
editorTitle: "OpenAI-compatible model ID",
|
||||
glassesEditTitle: "Edit compatible model ID",
|
||||
formatValue: (value) => value || "(not set)",
|
||||
description: "Exact model ID sent to the compatible provider. For glasses tools, choose a model that supports function calling.",
|
||||
});
|
||||
|
||||
export const veniceWebSearchSetting = new ConfigSettingBoolean({
|
||||
id: "venice-web-search",
|
||||
label: "Venice web search",
|
||||
storageKey: "assistant.veniceWebSearch",
|
||||
defaultValue: false,
|
||||
description: "Allow supported Venice models to search the web for current information. Shown only when OpenAI-compatible is the assistant model and its URL is the official Venice API. Search queries are processed by Venice's search service and may incur additional charges.",
|
||||
});
|
||||
|
||||
export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
id: "elevenlabs-api-key",
|
||||
label: "ElevenLabs key",
|
||||
@@ -1101,11 +1162,12 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
||||
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
|
||||
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
|
||||
if (provider === "local") return !isLocalModelReady();
|
||||
if (provider === "hermes") return assistantHermesUrlSetting.get().trim().length === 0;
|
||||
// Hermes and OpenAI-compatible stay selectable before they're configured:
|
||||
// their rows only appear in Settings once one is the selected model.
|
||||
return false;
|
||||
},
|
||||
description:
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
global.isIOS ? "Model used by the assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model; it never picks OpenAI-compatible or Hermes agent. Choosing either of those shows its settings below." : "Model used by the assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model; it never picks OpenAI-compatible or Hermes agent. Choosing either of those shows its settings below.",
|
||||
});
|
||||
|
||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
@@ -1119,6 +1181,21 @@ export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
description: "Mapbox public token (pk. prefix), used by the Navigate app for maps, geocoding, and directions.",
|
||||
});
|
||||
|
||||
/** Edited from the Music app's Spotify menu; a sign-in is tied to the client ID it was made with. */
|
||||
export const spotifyClientIdSetting = new ConfigSettingString({
|
||||
id: "spotify-client-id",
|
||||
label: "Client ID",
|
||||
storageKey: SPOTIFY_CLIENT_ID_KEY,
|
||||
defaultValue: "",
|
||||
editorTitle: "Spotify client ID (from your app in the Spotify Developer Dashboard)",
|
||||
glassesEditTitle: "Edit Spotify client ID",
|
||||
normalize: (value) => (value ?? "").trim(),
|
||||
validate: (value) => (!value || isValidSpotifyClientId(value) ? null
|
||||
: "A Spotify client ID is 32 letters and digits, shown on your app's page in the Spotify Developer Dashboard."),
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
description: "The client ID of your own Spotify developer app, which Faceclaw signs in with.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Staging buffer for the Terminal app's "Add connection" flow: the worker
|
||||
* asks the shell to open the phone text editor on this setting, the user
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
} from "../../native/local-llm";
|
||||
import {
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
@@ -42,13 +42,18 @@ import {
|
||||
assistantBridgePortSetting,
|
||||
assistantBridgeTokenSetting,
|
||||
assistantModelSetting,
|
||||
assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting,
|
||||
assistantOpenAiCompatibleUrlSetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
veniceWebSearchSetting,
|
||||
batteryDisplayModeSetting,
|
||||
brightnessSetting,
|
||||
autoBrightnessMinSetting,
|
||||
autoBrightnessMaxSetting,
|
||||
autoBrightnessCurveSetting,
|
||||
dataCollectionSetting,
|
||||
distanceUnitSetting,
|
||||
glassesBatteryVisibilitySetting,
|
||||
phoneBatteryVisibilitySetting,
|
||||
ringBatteryVisibilitySetting,
|
||||
@@ -93,6 +98,8 @@ import {
|
||||
import { wearBridge } from "../../native/wear-bridge";
|
||||
import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from "./settings-panel";
|
||||
import { uiFontPickerMenuItem } from "../font-picker";
|
||||
import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker";
|
||||
import { isVeniceApiBaseUrl } from "../../native/openai-chat";
|
||||
|
||||
/** The Settings app's master-detail panel (sections on the left, contents on the right). */
|
||||
export function createSettingsPanelLayer(): SettingsPanelLayer {
|
||||
@@ -141,22 +148,42 @@ function settingsSections(): SettingsSection[] {
|
||||
},
|
||||
{
|
||||
label: "Assistant",
|
||||
items: [
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
textSettingMenuItem(assistantHermesUrlSetting),
|
||||
textSettingMenuItem(assistantHermesKeySetting),
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
textSettingMenuItem(assistantBridgeTokenSetting),
|
||||
toggleSettingMenuItem(assistantAllowProactiveSetting),
|
||||
] : []),
|
||||
],
|
||||
// Re-read on every paint: provider-specific rows follow the selected
|
||||
// model (and, for Venice, the compatible URL).
|
||||
get items() {
|
||||
const selectedModel = assistantModelSetting.get();
|
||||
return [
|
||||
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
...(selectedModel === "hermes" ? [
|
||||
textSettingMenuItem(assistantHermesUrlSetting),
|
||||
textSettingMenuItem(assistantHermesKeySetting),
|
||||
] : []),
|
||||
...(selectedModel === "openai-compatible" ? [
|
||||
textSettingMenuItem(assistantOpenAiCompatibleUrlSetting),
|
||||
textSettingMenuItem(assistantOpenAiCompatibleKeySetting),
|
||||
compatibleModelDiscoveryMenuItem({
|
||||
getBaseUrl: () => assistantOpenAiCompatibleUrlSetting.get(),
|
||||
getApiKey: () => assistantOpenAiCompatibleKeySetting.get(),
|
||||
getModel: () => assistantOpenAiCompatibleModelSetting.get(),
|
||||
setModel: (model) => assistantOpenAiCompatibleModelSetting.set(model),
|
||||
}, assistantOpenAiCompatibleModelSetting),
|
||||
textSettingMenuItem(assistantOpenAiCompatibleModelSetting),
|
||||
...(isVeniceApiBaseUrl(assistantOpenAiCompatibleUrlSetting.get())
|
||||
? [toggleSettingMenuItem(veniceWebSearchSetting)] : []),
|
||||
] : []),
|
||||
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
textSettingMenuItem(assistantBridgeTokenSetting),
|
||||
toggleSettingMenuItem(assistantAllowProactiveSetting),
|
||||
] : []),
|
||||
];
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "API Keys",
|
||||
@@ -292,6 +319,8 @@ function customizationRows(): () => MenuItem[] {
|
||||
enumSettingMenuItem(timeFormatSetting),
|
||||
// Fahrenheit or Celsius, wherever weather temperatures show.
|
||||
enumSettingMenuItem(temperatureUnitSetting),
|
||||
// Miles or kilometers: Weather's wind speed and Navigate's distances.
|
||||
enumSettingMenuItem(distanceUnitSetting),
|
||||
];
|
||||
const bottomRows = [font, position, depth, statusBar, ...rest];
|
||||
const popupRows = [font, position, depth, statusBar, statusBarVisibility, windowBorder, ...rest];
|
||||
@@ -396,6 +425,29 @@ function localModelStatusText(): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
|
||||
* installs and manages; the row only reports whether it can be used.
|
||||
*/
|
||||
function systemModelMenuItem(): MenuItem {
|
||||
const statusText = (): string => {
|
||||
const state = localModelState();
|
||||
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
|
||||
};
|
||||
return {
|
||||
label: "On-phone model",
|
||||
description:
|
||||
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
|
||||
"Faster and more private than the cloud models, but simpler, with a short memory. " +
|
||||
"Used automatically when no API key is set.",
|
||||
onSelect: () => {},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||
function localModelMenuItem(): MenuItem {
|
||||
return {
|
||||
|
||||
+19
-6
@@ -14,14 +14,16 @@ export type ContinuousCaptureOptions = {
|
||||
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */
|
||||
recordAs?: string;
|
||||
/**
|
||||
* Transcribe with this provider instead of the Transcription Provider
|
||||
* setting, letting it detect the spoken language (Translate). While any
|
||||
* client asks for one, every capture uses it, and a shared capture running
|
||||
* another provider restarts on it.
|
||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||
* or "" for the provider to detect it), instead of the Transcription
|
||||
* Provider setting (Translate). While any client asks for one, every
|
||||
* capture uses it, and a shared capture running another restarts on it.
|
||||
*/
|
||||
multilingualProvider?: VoiceProvider;
|
||||
transcription?: TranscriptionChoice;
|
||||
};
|
||||
|
||||
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
|
||||
|
||||
export type LayerActions = {
|
||||
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
|
||||
requestRender: () => void;
|
||||
@@ -100,6 +102,15 @@ export interface Layer {
|
||||
readonly depth?: number;
|
||||
paintParts?(): Plane[];
|
||||
readonly paintOverBase?: boolean;
|
||||
/**
|
||||
* True when this layer stands in for `below`, a stacked layer directly
|
||||
* beneath it (or beneath another layer it stands in for): when this layer
|
||||
* calls paintBelow, the stack skips `below` and paints what is under it
|
||||
* instead. A submenu replacing the menu it opened from uses this, so the
|
||||
* hidden menu costs no plane and its dimUnderneath doesn't dim the content
|
||||
* a second time; popping the submenu still returns to it.
|
||||
*/
|
||||
replacesLayerBelow?(below: Layer): boolean;
|
||||
/**
|
||||
* Dim everything painted beneath this layer: a 0..1 brightness factor
|
||||
* applied to the pixels and glyphs of every plane below it in the stack
|
||||
@@ -324,7 +335,9 @@ export class LayerStack {
|
||||
return [ownPlane];
|
||||
}
|
||||
const dim = layer.dimUnderneath || 1;
|
||||
const below = this.paintLayer(layer.paintOverBase ? 0 : index - 1, dimSoFar * dim, raw);
|
||||
let belowIndex = layer.paintOverBase ? 0 : index - 1;
|
||||
while (belowIndex > 0 && layer.replacesLayerBelow?.(this.layers[belowIndex]!)) belowIndex--;
|
||||
const below = this.paintLayer(belowIndex, dimSoFar * dim, raw);
|
||||
return [...(!raw && dim < 1 ? dimPlanes(below, dim) : below), ownPlane];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { type GrayImage, type UiFont } from "../graphics/image";
|
||||
import { truncateText } from "../graphics/textwrap";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { lineStep } from "./metrics";
|
||||
|
||||
const HEADER_X = 24;
|
||||
const HEADER_Y = 20;
|
||||
const TITLE_GRAY = 200;
|
||||
const STATUS_GRAY = 110;
|
||||
|
||||
const SEGMENTS = 8;
|
||||
const SEGMENT_WIDTH = 4;
|
||||
const SEGMENT_GAP = 2;
|
||||
const METER_GAP = 10;
|
||||
const LIT_GRAY = 200;
|
||||
const UNLIT_GRAY = 45;
|
||||
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
|
||||
const TICK_MS = 200;
|
||||
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
|
||||
const DECAY = 0.5;
|
||||
|
||||
/**
|
||||
* A compact segmented mic-level meter for listening views. Levels arrive per
|
||||
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
|
||||
* asks for a repaint only when the number of lit segments changed and the
|
||||
* window is on screen.
|
||||
*/
|
||||
export class MicLevelMeter {
|
||||
readonly width = SEGMENTS * (SEGMENT_WIDTH + SEGMENT_GAP) - SEGMENT_GAP;
|
||||
private peak = 0;
|
||||
private shown = 0;
|
||||
private lit = 0;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
|
||||
this.peak = Math.max(this.peak, level);
|
||||
});
|
||||
this.timer = setInterval(() => {
|
||||
this.shown = Math.max(this.peak, this.shown * DECAY);
|
||||
this.peak = 0;
|
||||
const lit = Math.min(SEGMENTS, Math.round(this.shown * SEGMENTS));
|
||||
if (lit === this.lit) return;
|
||||
this.lit = lit;
|
||||
if (isVisible()) requestRender();
|
||||
}, TICK_MS);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
if (this.timer !== null) clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
|
||||
/** Draw the segments with their bottoms on `baseline`, `height` tall. */
|
||||
paint(image: GrayImage, x: number, baseline: number, height: number): void {
|
||||
for (let index = 0; index < SEGMENTS; index++) {
|
||||
const segmentX = x + index * (SEGMENT_WIDTH + SEGMENT_GAP);
|
||||
image.fillRect(segmentX, baseline - height, SEGMENT_WIDTH, height, index < this.lit ? LIT_GRAY : UNLIT_GRAY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The one-line header of the listening views (Transcribe, Translate):
|
||||
* "Title · status", the status truncated to fit, then the mic level meter at
|
||||
* the line's end when one is given (while listening). Returns the y at which
|
||||
* the view's body starts.
|
||||
*/
|
||||
export function paintListeningHeader(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
title: string,
|
||||
status: string,
|
||||
meter: MicLevelMeter | null,
|
||||
): number {
|
||||
let textRight = image.width - HEADER_X;
|
||||
if (meter) {
|
||||
const meterX = textRight - meter.width;
|
||||
// Roughly cap height, sitting on the text's baseline.
|
||||
meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6)));
|
||||
textRight = meterX - METER_GAP;
|
||||
}
|
||||
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
|
||||
if (status) {
|
||||
const statusX = HEADER_X + Math.ceil(font.measureText(title));
|
||||
image.drawText(font, statusX, HEADER_Y, truncateText(font, ` · ${status}`, textRight - statusX), STATUS_GRAY);
|
||||
}
|
||||
return HEADER_Y + lineStep(font) + 2;
|
||||
}
|
||||
+118
-42
@@ -9,12 +9,16 @@ import {
|
||||
dismissNotification,
|
||||
invokeNotificationAction,
|
||||
readActiveNotifications,
|
||||
readNotificationCustomView,
|
||||
readNotificationIconByKey,
|
||||
readNotificationLargeIcon,
|
||||
readNotificationPicture,
|
||||
type AndroidNotification,
|
||||
type AndroidNotificationAction,
|
||||
type NotificationImageResult,
|
||||
} from "../native/notification-icons";
|
||||
import { setNotificationSourceEnabled } from "../native/notification-sources";
|
||||
import { NOTIFICATION_ICON_SIZE } from "../native/notification-types";
|
||||
import { notificationEmptyMessage } from "../native/notification-access";
|
||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../util/render-freshness";
|
||||
import { type InputEvent } from "./gestures";
|
||||
@@ -26,7 +30,7 @@ const TITLE_X = 18;
|
||||
const TITLE_Y = 10;
|
||||
const LIST_TOP = 38;
|
||||
const CARD_X = 20;
|
||||
const ICON_SIZE = 24;
|
||||
const ICON_SIZE = NOTIFICATION_ICON_SIZE;
|
||||
const ICON_TEXT_GAP = 8;
|
||||
const CARD_TEXT_X = 10 + ICON_SIZE + ICON_TEXT_GAP;
|
||||
const CARD_GAP = 6;
|
||||
@@ -45,13 +49,18 @@ const DETAIL_TITLE_Y = 21;
|
||||
/** Gaps below the title's line box and below the icon/app-name header row. */
|
||||
const DETAIL_HEADER_GAP = 3;
|
||||
const DETAIL_BODY_GAP = 4;
|
||||
/** Text lines an attached picture leaves room for above it (when there are that many); it takes the rest. */
|
||||
/** Untitled text lines that attached images (custom layout, picture) leave room for above them, when there are that many; they take the rest. */
|
||||
const DETAIL_PICTURE_TEXT_LINES = 2;
|
||||
/** Gap between the text and an attached picture, and the space kept below the picture. */
|
||||
const DETAIL_PICTURE_GAP = 6;
|
||||
const DETAIL_PICTURE_BOTTOM_MARGIN = 12;
|
||||
/** An attached picture with less height than this to fit in (a large UI font) is left out. */
|
||||
const DETAIL_PICTURE_MIN_HEIGHT = 32;
|
||||
/** Likewise a custom layout; Dexcom's reading is about 40 high. */
|
||||
const DETAIL_CUSTOM_VIEW_MIN_HEIGHT = 16;
|
||||
/** A large icon (sender's avatar, site's logo): at the right of a list card, left of the detail view's title. */
|
||||
const LARGE_ICON_SIZE = 32;
|
||||
const LARGE_ICON_GAP = 8;
|
||||
// Disable-source confirmation: actions on top, the explanation below them.
|
||||
const CONFIRM_X = 24;
|
||||
const CONFIRM_Y = 8;
|
||||
@@ -67,14 +76,27 @@ function iconForNotification(key: string): GrayImage | null {
|
||||
return icon;
|
||||
}
|
||||
|
||||
/** Attached picture for a paint pass, fitting maxWidth x maxHeight; allow-stale like iconForNotification. */
|
||||
function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null {
|
||||
if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null;
|
||||
const { picture, stale } = readNotificationPicture(notification, maxWidth, maxHeight, renderPassAllowsStaleData());
|
||||
if (stale) {
|
||||
/** A notification image read for a paint pass (allow-stale like iconForNotification). */
|
||||
function imageForPaint(
|
||||
read: (allowStale: boolean) => NotificationImageResult,
|
||||
): NotificationImageResult {
|
||||
const result = read(renderPassAllowsStaleData());
|
||||
if (result.stale) {
|
||||
noteStaleDataUsed();
|
||||
}
|
||||
return picture;
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Attached picture for a paint pass, fitting maxWidth x maxHeight. */
|
||||
function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null {
|
||||
if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null;
|
||||
return imageForPaint((allowStale) => readNotificationPicture(notification, maxWidth, maxHeight, allowStale)).image;
|
||||
}
|
||||
|
||||
/** Large icon for a paint pass, LARGE_ICON_SIZE square at most. */
|
||||
function largeIconForNotification(notification: AndroidNotification): GrayImage | null {
|
||||
if (!notification.hasLargeIcon) return null;
|
||||
return imageForPaint((allowStale) => readNotificationLargeIcon(notification, LARGE_ICON_SIZE, allowStale)).image;
|
||||
}
|
||||
|
||||
type CardLayout = {
|
||||
@@ -152,7 +174,8 @@ export class NotificationsListLayer implements Layer {
|
||||
// Icons are only resolved for cards actually drawn, so a long list
|
||||
// does not fetch icons for everything below the fold.
|
||||
const icon = iconForNotification(layout.notification.key);
|
||||
drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon);
|
||||
const largeIcon = largeIconForNotification(layout.notification);
|
||||
drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon, largeIcon);
|
||||
}
|
||||
cursorY += layout.height + CARD_GAP;
|
||||
if (cursorY > listBottom + 80) break;
|
||||
@@ -356,8 +379,10 @@ function buildNotificationCardLayout(
|
||||
font: UiFont,
|
||||
notification: AndroidNotification,
|
||||
selected: boolean,
|
||||
cardTextWidth: number,
|
||||
fullTextWidth: number,
|
||||
): CardLayout {
|
||||
// The large icon sits at the card's right, beside all its text.
|
||||
const cardTextWidth = notification.hasLargeIcon ? fullTextWidth - LARGE_ICON_SIZE - LARGE_ICON_GAP : fullTextWidth;
|
||||
const lines: string[] = [];
|
||||
const time = formatRelativeTime(notification.postTime);
|
||||
const title = notificationTitle(notification);
|
||||
@@ -383,7 +408,7 @@ function buildNotificationCardLayout(
|
||||
return {
|
||||
notification,
|
||||
lines,
|
||||
height: Math.max(ICON_SIZE + 18, 12 + lines.length * lineStep(font)),
|
||||
height: Math.max(ICON_SIZE + 18, notification.hasLargeIcon ? LARGE_ICON_SIZE + 16 : 0, 12 + lines.length * lineStep(font)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -397,6 +422,7 @@ function drawNotificationCard(
|
||||
selected: boolean,
|
||||
focused: boolean,
|
||||
icon: GrayImage | null,
|
||||
largeIcon: GrayImage | null,
|
||||
): void {
|
||||
// Match the shared menu highlight: faint border + black fill when
|
||||
// unselected; bright border when selected; the non-black fill only appears
|
||||
@@ -408,6 +434,9 @@ function drawNotificationCard(
|
||||
if (icon) {
|
||||
image.bitBlt(icon, x + 10, y + 8, { transparentZero: true });
|
||||
}
|
||||
if (largeIcon) {
|
||||
image.bitBlt(largeIcon, x + width - 10 - LARGE_ICON_SIZE + ((LARGE_ICON_SIZE - largeIcon.width) >> 1), y + 8);
|
||||
}
|
||||
for (let index = 0; index < layout.lines.length; index++) {
|
||||
const value = index === 0 ? 140 : selected ? 235 : 185;
|
||||
image.drawText(font, x + CARD_TEXT_X, y + 7 + index * lineStep(font), layout.lines[index]!, value);
|
||||
@@ -437,18 +466,35 @@ function drawDetailContent(
|
||||
): void {
|
||||
const contentX = NOTIFICATION_CONTENT_X;
|
||||
const contentWidth = notificationContentWidth(width);
|
||||
const bodyTop = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`);
|
||||
const step = lineStep(font);
|
||||
const bottom = height - DETAIL_PICTURE_BOTTOM_MARGIN;
|
||||
let top = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`);
|
||||
|
||||
// The title, beside the large icon (a sender's avatar, a site's logo) when
|
||||
// there is one. A picture or custom layout can be the whole notification
|
||||
// (Dexcom's has no text), so either stands in for the "(untitled)" placeholder.
|
||||
const hasImages = Boolean(notification.hasPicture || notification.hasCustomView);
|
||||
const title = notification.title || (hasImages ? "" : "(untitled)");
|
||||
const avatarSize = notification.hasLargeIcon ? LARGE_ICON_SIZE : 0;
|
||||
const titleX = avatarSize ? contentX + avatarSize + LARGE_ICON_GAP : contentX;
|
||||
const titleWidth = contentX + contentWidth - titleX;
|
||||
const titleLines = title ? wrapText(font, title, titleWidth) : [];
|
||||
let titleHeight = 0;
|
||||
if (avatarSize) {
|
||||
const avatar = largeIconForNotification(notification);
|
||||
if (avatar) image.bitBlt(avatar, contentX + ((avatarSize - avatar.width) >> 1), top + ((avatarSize - avatar.height) >> 1));
|
||||
titleHeight = avatarSize;
|
||||
}
|
||||
if (titleLines.length) {
|
||||
// A title shorter than the avatar is centered on it.
|
||||
const titleTop = top + Math.max(0, (avatarSize - titleLines.length * step) >> 1);
|
||||
const shown = drawClippedLines(image, font, titleLines, titleX, titleTop, titleWidth, ((bottom - top) / step) | 0, 230);
|
||||
titleHeight = Math.max(titleHeight, shown * step);
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
// A picture can be the whole notification (Dexcom's has no text), so it
|
||||
// stands in for the "(untitled)" placeholder.
|
||||
const hasTitle = Boolean(notification.title) || !notification.hasPicture;
|
||||
if (hasTitle) {
|
||||
lines.push(...wrapText(font, notification.title || "(untitled)", contentWidth));
|
||||
}
|
||||
const body = detailNotificationBody(notification);
|
||||
if (body) {
|
||||
if (lines.length) lines.push("");
|
||||
lines.push(...wrapText(font, body, contentWidth));
|
||||
}
|
||||
const meta = [notification.subText, notification.infoText, notification.summaryText].filter(Boolean).join(" ");
|
||||
@@ -456,36 +502,65 @@ function drawDetailContent(
|
||||
if (lines.length) lines.push("");
|
||||
lines.push(...wrapText(font, meta, contentWidth));
|
||||
}
|
||||
// Text follows the title after a blank line (just a gap beside an avatar); images after a gap.
|
||||
if (titleHeight) {
|
||||
top += titleHeight + (lines.length && !avatarSize ? step : DETAIL_PICTURE_GAP);
|
||||
}
|
||||
|
||||
const step = lineStep(font);
|
||||
const textValue = (index: number) => index === 0 && hasTitle ? 230 : 190;
|
||||
const reservedTextHeight = Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step;
|
||||
const picture = pictureForNotification(notification, contentWidth,
|
||||
height - DETAIL_PICTURE_BOTTOM_MARGIN - bodyTop - reservedTextHeight - (reservedTextHeight ? DETAIL_PICTURE_GAP : 0));
|
||||
if (picture) {
|
||||
// The text gets whatever the picture leaves, above it, with its last
|
||||
// shown line marked when some is cut off.
|
||||
const roomForText = height - DETAIL_PICTURE_BOTTOM_MARGIN - picture.height - DETAIL_PICTURE_GAP - bodyTop;
|
||||
const shownLines = Math.min(lines.length, Math.max(0, (roomForText / step) | 0));
|
||||
for (let index = 0; index < shownLines; index++) {
|
||||
const line = index === shownLines - 1 && lines.length > shownLines
|
||||
? truncateText(font, lines[index] + "...", contentWidth) : lines[index]!;
|
||||
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
|
||||
// The custom layout, then the picture, take what is left below the title,
|
||||
// or below the first few lines of text when there is no title.
|
||||
const reservedTextHeight = titleHeight ? 0 : Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step;
|
||||
const roomForImages = bottom - top - (reservedTextHeight ? reservedTextHeight + DETAIL_PICTURE_GAP : 0);
|
||||
let customView: GrayImage | null = null;
|
||||
if (notification.hasCustomView && roomForImages >= DETAIL_CUSTOM_VIEW_MIN_HEIGHT) {
|
||||
const result = imageForPaint((allowStale) => readNotificationCustomView(notification, contentWidth, roomForImages, allowStale));
|
||||
customView = result.image;
|
||||
if (!customView && !result.stale && notification.customText?.length) {
|
||||
// It couldn't be drawn: its text is better than nothing.
|
||||
lines.unshift(...wrapText(font, notification.customText.join(" "), contentWidth), ...(lines.length ? [""] : []));
|
||||
}
|
||||
image.bitBlt(picture, contentX, bodyTop + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0));
|
||||
return;
|
||||
}
|
||||
const picture = pictureForNotification(notification, contentWidth,
|
||||
roomForImages - (customView ? customView.height + DETAIL_PICTURE_GAP : 0));
|
||||
|
||||
const maxLines = Math.max(1, ((height - bodyTop - step) / step) | 0);
|
||||
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
|
||||
const line = lines[index]!;
|
||||
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
|
||||
// The text gets whatever the images leave, above them, with its last shown
|
||||
// line marked when some is cut off.
|
||||
const imagesHeight = (customView ? customView.height : 0) + (picture ? picture.height : 0)
|
||||
+ (customView && picture ? DETAIL_PICTURE_GAP : 0);
|
||||
const roomForText = bottom - top - (imagesHeight ? imagesHeight + DETAIL_PICTURE_GAP : 0);
|
||||
const shownLines = drawClippedLines(image, font, lines, contentX, top, contentWidth, (roomForText / step) | 0, 190);
|
||||
let imageTop = top + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0);
|
||||
if (customView) {
|
||||
image.bitBlt(customView, contentX, imageTop);
|
||||
imageTop += customView.height + DETAIL_PICTURE_GAP;
|
||||
}
|
||||
if (lines.length > maxLines) {
|
||||
image.drawText(font, contentX, height - 24, "...", 140);
|
||||
if (picture) {
|
||||
image.bitBlt(picture, contentX, imageTop);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw up to maxLines of lines from (x, y), marking the last one drawn with
|
||||
* "..." when some are left out. Returns how many were drawn.
|
||||
*/
|
||||
function drawClippedLines(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
lines: string[],
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
maxLines: number,
|
||||
value: number,
|
||||
): number {
|
||||
const shown = Math.min(lines.length, Math.max(0, maxLines));
|
||||
for (let index = 0; index < shown; index++) {
|
||||
const line = index === shown - 1 && lines.length > shown ? truncateText(font, lines[index] + "...", width) : lines[index]!;
|
||||
image.drawText(font, x, y + index * lineStep(font), line, value);
|
||||
}
|
||||
return shown;
|
||||
}
|
||||
|
||||
/** Left edge of a notification detail view's content column. */
|
||||
export const NOTIFICATION_CONTENT_X = DETAIL_CONTENT_X;
|
||||
|
||||
@@ -577,5 +652,6 @@ function detailNotificationBody(notification: AndroidNotification): string {
|
||||
}
|
||||
|
||||
function notificationTitle(notification: AndroidNotification): string {
|
||||
return notification.title || notification.text || notification.summaryText || notification.appName || notification.packageName || "(untitled)";
|
||||
return notification.title || notification.text || notification.customText?.join(" ") || notification.summaryText
|
||||
|| notification.appName || notification.packageName || "(untitled)";
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import { MenuLayer, type MenuItem } from "./menu";
|
||||
import { EditTextSettingLayer } from "./dashboard-settings";
|
||||
import type { LayerContext } from "./layers";
|
||||
import {
|
||||
discoverOpenAiCompatibleModels,
|
||||
type DiscoveredCompatibleModel,
|
||||
} from "../native/openai-chat-models";
|
||||
|
||||
export type ModelDiscoverySettings = {
|
||||
getBaseUrl(): string;
|
||||
getApiKey(): string;
|
||||
getModel(): string;
|
||||
setModel(model: string): void;
|
||||
};
|
||||
|
||||
type Discover = typeof discoverOpenAiCompatibleModels;
|
||||
type PickerState = "loading" | "models" | "error";
|
||||
|
||||
const DISCOVERY_EXPLANATION =
|
||||
"Discover models queries only this server's /models endpoint. Choose an exact model ID, or keep using manual entry. This does not send an inference request.";
|
||||
|
||||
/** A cancellable, on-demand model picker embedded in Faceclaw's existing menus. */
|
||||
export class OpenAiCompatibleModelPicker extends MenuLayer {
|
||||
private state: PickerState = "loading";
|
||||
private models: DiscoveredCompatibleModel[] = [];
|
||||
private error = "";
|
||||
private showOffline = false;
|
||||
private requestGeneration = 0;
|
||||
private endpointKey = "";
|
||||
private removed = false;
|
||||
private readonly modelIdByRow = new Map<MenuItem, string>();
|
||||
|
||||
constructor(
|
||||
private readonly settings: ModelDiscoverySettings,
|
||||
private readonly modelSetting: ConstructorParameters<typeof EditTextSettingLayer>[0],
|
||||
private readonly discover: Discover = discoverOpenAiCompatibleModels,
|
||||
viewport: { width: number; height: number } = { width: 640, height: 480 },
|
||||
) {
|
||||
super("Discover models", [], discoveryLayout(viewport));
|
||||
}
|
||||
|
||||
open(ctx: LayerContext): void {
|
||||
ctx.stack.push(this);
|
||||
void this.refresh(ctx);
|
||||
}
|
||||
|
||||
private currentEndpoint(): string {
|
||||
return `${this.settings.getBaseUrl().trim()}\n${this.settings.getApiKey().trim()}`;
|
||||
}
|
||||
|
||||
private async refresh(ctx: LayerContext, forceRefresh = false): Promise<void> {
|
||||
const generation = ++this.requestGeneration;
|
||||
const endpointKey = this.currentEndpoint();
|
||||
this.endpointKey = endpointKey;
|
||||
this.state = "loading";
|
||||
this.error = "";
|
||||
this.updateRows();
|
||||
try {
|
||||
const models = await this.discover({
|
||||
baseUrl: this.settings.getBaseUrl(),
|
||||
apiKey: this.settings.getApiKey(),
|
||||
forceRefresh,
|
||||
});
|
||||
if (!this.isActive(generation)) return;
|
||||
if (endpointKey !== this.currentEndpoint()) {
|
||||
this.error = "API URL or key changed; discover again";
|
||||
this.state = "error";
|
||||
} else {
|
||||
this.models = models;
|
||||
this.state = "models";
|
||||
}
|
||||
} catch (error) {
|
||||
if (!this.isActive(generation)) return;
|
||||
this.error = endpointKey !== this.currentEndpoint()
|
||||
? "API URL or key changed; discover again"
|
||||
: error instanceof Error ? error.message : String(error);
|
||||
this.state = "error";
|
||||
}
|
||||
this.updateRows();
|
||||
ctx.actions.requestRender();
|
||||
}
|
||||
|
||||
private isActive(generation: number): boolean {
|
||||
// A temporary overlay may sit above this still-open picker. Removal, not
|
||||
// top-of-stack status, determines whether the pending result is obsolete.
|
||||
return !this.removed && generation === this.requestGeneration;
|
||||
}
|
||||
|
||||
private updateRows(): void {
|
||||
this.modelIdByRow.clear();
|
||||
const rows: MenuItem[] = [];
|
||||
if (this.state === "loading") {
|
||||
rows.push({ label: "Discovering models...", disabled: true, description: DISCOVERY_EXPLANATION, onSelect: () => {} });
|
||||
rows.push({
|
||||
label: "Cancel discovery",
|
||||
description: "Ignore the pending result and return to Assistant settings.",
|
||||
onSelect: (innerCtx) => innerCtx.stack.pop(),
|
||||
});
|
||||
} else if (this.state === "error") {
|
||||
rows.push({
|
||||
label: `Failed: ${truncateModelLabel(this.error || "Discovery failed", 40)}`,
|
||||
disabled: true,
|
||||
description: `${this.error}. Manual model entry is still available.`,
|
||||
onSelect: () => {},
|
||||
});
|
||||
rows.push({
|
||||
label: "Try again",
|
||||
description: "Query this server's /models endpoint again without changing the saved model.",
|
||||
onSelect: (innerCtx) => { void this.refresh(innerCtx, true); },
|
||||
});
|
||||
} else {
|
||||
const saved = this.settings.getModel().trim();
|
||||
const listed = this.models.filter((model) => this.showOffline || !model.offline || model.id === saved);
|
||||
const savedModel = saved ? this.models.find((model) => model.id === saved) : undefined;
|
||||
if (saved && !savedModel) {
|
||||
rows.push(this.modelRow({
|
||||
id: saved, name: saved, supportsTools: null, offline: false, deprecated: false, beta: false,
|
||||
}, "saved; not listed"));
|
||||
}
|
||||
if (!listed.length) {
|
||||
rows.push({
|
||||
label: this.models.length ? "No online models listed" : "No models listed",
|
||||
disabled: true,
|
||||
description: "Keep the saved model or use manual entry.",
|
||||
onSelect: () => {},
|
||||
});
|
||||
} else {
|
||||
rows.push(...listed.map((model) => this.modelRow(model)));
|
||||
}
|
||||
if (this.models.some((model) => model.offline)) {
|
||||
rows.push({
|
||||
label: this.showOffline ? "Hide offline models" : `Show offline (${this.models.filter((model) => model.offline).length})`,
|
||||
description: "Offline models normally fail inference until the provider restores them.",
|
||||
onSelect: (innerCtx) => {
|
||||
this.showOffline = !this.showOffline;
|
||||
this.updateRows();
|
||||
innerCtx.actions.requestRender();
|
||||
},
|
||||
});
|
||||
}
|
||||
rows.push({
|
||||
label: "Refresh models",
|
||||
description: "Fetch a fresh catalog. This never changes the selected model by itself.",
|
||||
onSelect: (innerCtx) => { void this.refresh(innerCtx, true); },
|
||||
});
|
||||
}
|
||||
rows.push({
|
||||
label: "Enter model manually",
|
||||
description: "Use an exact model ID when discovery is unavailable or the desired model is not listed.",
|
||||
onSelect: (innerCtx) => {
|
||||
this.requestGeneration++; // Any late discovery result belongs to the closed picker.
|
||||
void innerCtx.actions.startTextSettingEdit(this.modelSetting);
|
||||
innerCtx.stack.popThrough(this);
|
||||
innerCtx.stack.push(new EditTextSettingLayer(this.modelSetting));
|
||||
},
|
||||
});
|
||||
const current = this.settings.getModel().trim();
|
||||
const selected = Math.max(0, rows.findIndex((row) => this.modelIdByRow.get(row) === current));
|
||||
this.setItems(rows, selected);
|
||||
}
|
||||
|
||||
private modelRow(model: DiscoveredCompatibleModel, extraLabel?: string): MenuItem {
|
||||
const flags: string[] = [];
|
||||
if (model.supportsTools === true) flags.push("tools");
|
||||
else if (model.supportsTools === false) flags.push("no tools");
|
||||
else flags.push("tools unknown");
|
||||
if (model.offline) flags.push("offline");
|
||||
if (model.deprecated) flags.push("deprecated");
|
||||
if (model.beta) flags.push("beta");
|
||||
if (extraLabel) flags.push(extraLabel);
|
||||
const selected = this.settings.getModel().trim() === model.id ? " *" : "";
|
||||
const displayName = model.name === model.id ? model.id : `${model.name} (${model.id})`;
|
||||
const toolFlag = model.supportsTools === true ? "tools" : model.supportsTools === false ? "no tools" : "tools?";
|
||||
const visibleFlags = [toolFlag];
|
||||
if (model.offline) visibleFlags.push("offline");
|
||||
if (model.deprecated) visibleFlags.push("old");
|
||||
if (model.beta) visibleFlags.push("beta");
|
||||
if (extraLabel) visibleFlags.push("saved only");
|
||||
const flagSuffix = ` [${visibleFlags.join("/")}]`;
|
||||
const row: MenuItem = {
|
||||
label: `${truncateModelLabel(displayName, Math.max(12, 44 - flagSuffix.length))}${flagSuffix}${selected}`,
|
||||
description: `${model.name}: ${model.id} — ${flags.join(", ")}. Tool support unknown means the server did not advertise it; test before relying on glasses actions.`,
|
||||
onSelect: (innerCtx) => {
|
||||
// Re-read immediately before saving; a late catalog never overwrites a
|
||||
// selection made against a different endpoint.
|
||||
if (this.settings.getBaseUrl().trim() + "\n" + this.settings.getApiKey().trim() !== this.endpointKey) return;
|
||||
this.settings.setModel(model.id);
|
||||
innerCtx.stack.popThrough(this);
|
||||
},
|
||||
};
|
||||
this.modelIdByRow.set(row, model.id);
|
||||
return row;
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.requestGeneration++; // Invalidate the pending result immediately.
|
||||
}
|
||||
}
|
||||
|
||||
/** Kept separate so tests can verify the normal Faceclaw modal sizing contract. */
|
||||
function discoveryLayout(viewport: { width: number; height: number }): ConstructorParameters<typeof MenuLayer>[2] {
|
||||
return {
|
||||
x: "center",
|
||||
y: 8,
|
||||
width: Math.min(320, viewport.width - 40),
|
||||
minHeight: Math.min(160, viewport.height - 40),
|
||||
maxHeight: Math.min(240, viewport.height - 40),
|
||||
depth: 4,
|
||||
};
|
||||
}
|
||||
|
||||
function truncateModelLabel(value: string, maxLength = 44): string {
|
||||
if (value.length <= maxLength) return value;
|
||||
const tailLength = Math.min(19, Math.max(4, maxLength - 8));
|
||||
const headLength = Math.max(1, maxLength - tailLength - 3);
|
||||
return `${value.slice(0, headLength)}...${value.slice(-tailLength)}`;
|
||||
}
|
||||
|
||||
/** One stable Assistant settings row; discovery state belongs to the pushed picker. */
|
||||
export function compatibleModelDiscoveryMenuItem(
|
||||
settings: ModelDiscoverySettings,
|
||||
modelSetting: ConstructorParameters<typeof EditTextSettingLayer>[0],
|
||||
discover: Discover = discoverOpenAiCompatibleModels,
|
||||
): MenuItem {
|
||||
return {
|
||||
label: "Discover models",
|
||||
description: DISCOVERY_EXPLANATION,
|
||||
disabled: () => !settings.getBaseUrl().trim(),
|
||||
onSelect: (ctx) => new OpenAiCompatibleModelPicker(settings, modelSetting, discover, ctx.stack.getBaseSize()).open(ctx),
|
||||
};
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { truncateText } from "../../graphics/textwrap";
|
||||
import { activeAmbientCards } from "./ambient-cards";
|
||||
import { BATTERY_ICON_WIDTH, DENSE_BATTERY_BLOCK_HEIGHT, drawBattery, drawDenseBatteries, type BatteryDevice } from "../../graphics/battery";
|
||||
import { ALL_NOTIFICATIONS, readActiveNotificationIcons } from "../../native/notification-icons";
|
||||
import { NOTIFICATION_ICON_SIZE } from "../../native/notification-types";
|
||||
import { readPhoneBatteryState } from "../../native/phone-battery";
|
||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
|
||||
import { renderIcon, renderIconWithGlyph, type IconActivity, type IconName } from "../../graphics/icons";
|
||||
@@ -430,7 +431,6 @@ function statusBarPlacement(state: ShellChromeState): StatusBarPlacement | null
|
||||
return { left, right, top, contentTop: atBottom ? top + 1 : top, divider };
|
||||
}
|
||||
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const NOTIFICATION_ICON_STEP = NOTIFICATION_ICON_SIZE + 4;
|
||||
// Notification icons' top, in the top bar's layout (where any status icon's
|
||||
// selection box starts, a row above).
|
||||
|
||||
@@ -192,12 +192,13 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
drawIcon: options.drawIcon ?? windowIcon(options.icon, options.iconLetter),
|
||||
handleInput: async (event, frameId) => {
|
||||
// The shell opened its system menu over this window; close our own
|
||||
// context menu so the two never stack. Never forwarded to app layers.
|
||||
// context menu (and any window submenus open from it) so the two never
|
||||
// stack. Never forwarded to app layers.
|
||||
if (event.type === "system-menu-opened") {
|
||||
options.onSystemMenuOpened?.();
|
||||
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
||||
requestRender();
|
||||
}
|
||||
let closedMenu = false;
|
||||
while (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) closedMenu = true;
|
||||
if (closedMenu) requestRender();
|
||||
return;
|
||||
}
|
||||
// The default tap-then-hold response: the window menu (or the system
|
||||
|
||||
Loaded 100 of 218 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user