forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes
This commit is contained in:
commit
fcea0b369a
218 files changed
+14438
-2259
No files matched your search
@@ -4,7 +4,7 @@ import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { ASSISTANT_MODEL_CHOICES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, assistantOpenAiCompatibleKeySetting, assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext } from "../../ui/layers";
|
||||
import type { MenuItem } from "../../ui/menu";
|
||||
@@ -102,7 +102,7 @@ class AiChatLayer implements Layer {
|
||||
}
|
||||
case "long-press":
|
||||
if (!this.conversations.current().session?.isTurnActive()) {
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings."); return; }
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(shell.assistantSetupHint()); return; }
|
||||
void this.draft.start();
|
||||
}
|
||||
return;
|
||||
@@ -133,7 +133,12 @@ class AiChatLayer implements Layer {
|
||||
onSelect: (pickCtx) => { if (this.conversations.configure(model, record.reasoning)) pickCtx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||
const llm = resolveAssistantModel(record.model, {
|
||||
anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
return external || busy() || !llm?.effort;
|
||||
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||
|
||||
@@ -7,13 +7,17 @@
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantModelSetting,
|
||||
assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting,
|
||||
assistantOpenAiCompatibleUrlSetting,
|
||||
openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
|
||||
@@ -22,6 +26,9 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
if (!resolved) return Promise.resolve(null);
|
||||
|
||||
@@ -40,8 +47,11 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
case "openai":
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "openai-compatible":
|
||||
streamOpenAiCompatibleChat({ ...options, baseUrl: resolved.baseUrl ?? "" });
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
streamLocalModel(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
|
||||
@@ -14,6 +14,9 @@ import {
|
||||
const MESSAGE_SIZES = [250, 500, 1000, 2000, 3800];
|
||||
const WINDOW_SIZES = [1, 2, 3, 4, 6];
|
||||
const LINK_MODES = ["Current", "Request HIGH", "Request 2M", "HIGH + 2M"];
|
||||
// CoreBluetooth has no connection-priority or PHY request API (iOS's
|
||||
// prepareBenchmarkLink is a no-op), so iOS always runs on the current link.
|
||||
const LINK_REQUESTS_SUPPORTED = !global.isIOS;
|
||||
const DURATION_MS = 15_000;
|
||||
// After painting the "running" screen, wait for that frame to reach the
|
||||
// glasses before the benchmark blocks normal image sends — otherwise the
|
||||
@@ -40,7 +43,7 @@ type ConfigRow = "size" | "window" | "link" | "start";
|
||||
export class BandwidthBenchmarkLayer implements Layer {
|
||||
private phase: Phase = "config";
|
||||
private readonly menu = new Menu<ConfigRow>({
|
||||
items: ["size", "window", "link", "start"],
|
||||
items: LINK_REQUESTS_SUPPORTED ? ["size", "window", "link", "start"] : ["size", "window", "start"],
|
||||
wrap: false,
|
||||
rowGap: 1,
|
||||
highlight: { radius: 4 },
|
||||
@@ -134,7 +137,8 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
font,
|
||||
LIST_X,
|
||||
HEADER_HEIGHT + 30,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}, ${LINK_MODES[this.linkModeIndex]}`,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}`
|
||||
+ (LINK_REQUESTS_SUPPORTED ? `, ${LINK_MODES[this.linkModeIndex]}` : ""),
|
||||
150,
|
||||
);
|
||||
image.drawText(font, LIST_X, HEADER_HEIGHT + 52, "Display updates pause until the run finishes.", 110);
|
||||
@@ -161,8 +165,11 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
`Throughput: ${(payloadRate / 1024).toFixed(2)} KB/s payload (${(wireRate / 1024).toFixed(2)} KB/s wire)`,
|
||||
`Messages: ${r.messagesAcked}/${r.messagesSent} acked, ${r.timeouts} timeouts`,
|
||||
`Elapsed: ${seconds.toFixed(1)} s` + (r.aborted ? " (aborted early)" : ""),
|
||||
`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}`,
|
||||
];
|
||||
if (LINK_REQUESTS_SUPPORTED) {
|
||||
const phy = r.leftPhy || r.rightPhy ? `; PHY L ${r.leftPhy ?? "?"}, R ${r.rightPhy ?? "?"}` : "";
|
||||
lines.push(`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}${phy}`);
|
||||
}
|
||||
for (const line of lines) {
|
||||
image.drawText(font, LIST_X, y, line, 190);
|
||||
y += 20;
|
||||
|
||||
@@ -51,24 +51,18 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Buzzer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Glasses sound playback is not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new BuzzerDemoLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Accelerometer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new AccelerometerDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Light sensor",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
@@ -83,8 +77,6 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Microphones",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: openMicrophones,
|
||||
},
|
||||
{
|
||||
@@ -131,8 +123,6 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc
|
||||
},
|
||||
{
|
||||
label: "Show resource usage",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new ResourceUsageLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
|
||||
@@ -4,25 +4,23 @@ import { clamp } from "../../util/numeric-util";
|
||||
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
|
||||
declare const com: any;
|
||||
import {
|
||||
MEMORY_SERIES,
|
||||
formatMb,
|
||||
sampleResourceUsage,
|
||||
type ResourceSample,
|
||||
} from "../../native/resource-usage";
|
||||
|
||||
const SAMPLE_INTERVAL_MS = 1_000;
|
||||
const HISTORY_MS = 60_000;
|
||||
const MAX_SAMPLES = Math.ceil(HISTORY_MS / SAMPLE_INTERVAL_MS) + 2;
|
||||
/** Line brightness per memory series, headline first. */
|
||||
const MEMORY_SHADES = [240, 155, 80];
|
||||
|
||||
type ResourceSample = {
|
||||
atMs: number;
|
||||
rssMb: number;
|
||||
nativeMb: number;
|
||||
javaMb: number;
|
||||
javaCommittedMb: number;
|
||||
cpuPercent: number;
|
||||
threads: number;
|
||||
};
|
||||
type ChartSample = ResourceSample & { cpuPercent: number };
|
||||
|
||||
type Series = {
|
||||
value: (sample: ResourceSample) => number;
|
||||
value: (sample: ChartSample) => number;
|
||||
shade: number;
|
||||
};
|
||||
|
||||
@@ -30,9 +28,10 @@ type Series = {
|
||||
* One-minute process monitor. Sampling deliberately continues while another
|
||||
* window is foreground so switching back reveals the resource cost of that
|
||||
* workload. Repaints remain visibility-gated to avoid creating hidden frames.
|
||||
* Which memory figures are charted is up to the platform (MEMORY_SERIES).
|
||||
*/
|
||||
export class ResourceUsageLayer implements Layer {
|
||||
private readonly samples: ResourceSample[] = [];
|
||||
private readonly samples: ChartSample[] = [];
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private removed = false;
|
||||
private previousElapsedMs = 0;
|
||||
@@ -68,34 +67,34 @@ export class ResourceUsageLayer implements Layer {
|
||||
return image;
|
||||
}
|
||||
|
||||
const cpuText = `CPU ${formatPercent(latest.cpuPercent)} ${latest.threads} threads`;
|
||||
image.drawText(font, width - 16 - font.measureText(cpuText), 5, cpuText, 200);
|
||||
image.drawText(font, 16, 21, this.sampleError || latest.status, this.sampleError ? 170 : 140);
|
||||
|
||||
const memorySeries = MEMORY_SERIES.map((label, index) => ({
|
||||
label,
|
||||
shade: MEMORY_SHADES[index] ?? 80,
|
||||
value: (sample: ChartSample) => sample.memoryMb[index] ?? 0,
|
||||
}));
|
||||
const memoryPeak = Math.max(...this.samples.flatMap((sample) => sample.memoryMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
const legendLeft = drawLegend(
|
||||
image,
|
||||
font,
|
||||
width,
|
||||
48,
|
||||
memorySeries.map((series) => [`${series.label} ${formatMb(series.value(latest))}`, series.shade] as const),
|
||||
);
|
||||
const scaleLabel = `Memory 0–${formatMb(memoryScale)}`;
|
||||
const shortScaleLabel = `0–${formatMb(memoryScale)}`;
|
||||
image.drawText(
|
||||
font,
|
||||
16,
|
||||
21,
|
||||
`RSS ${formatMb(latest.rssMb)} native ${formatMb(latest.nativeMb)} Java ${formatMb(latest.javaMb)}/${formatMb(latest.javaCommittedMb)}`,
|
||||
190,
|
||||
48,
|
||||
16 + font.measureText(scaleLabel) + 8 <= legendLeft ? scaleLabel : shortScaleLabel,
|
||||
140,
|
||||
);
|
||||
image.drawText(
|
||||
font,
|
||||
width - 16 - font.measureText(`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`),
|
||||
47 - font.lineHeight,
|
||||
`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`,
|
||||
155,
|
||||
);
|
||||
|
||||
const memoryPeak = Math.max(...this.samples.map((sample) => sample.rssMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
image.drawText(font, 16, 48, `Memory 0–${formatMb(memoryScale)}`, 140);
|
||||
drawLegend(image, font, width, 48, [
|
||||
["RSS", 240],
|
||||
["native", 155],
|
||||
["Java", 80],
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, [
|
||||
{ value: (sample) => sample.rssMb, shade: 240 },
|
||||
{ value: (sample) => sample.nativeMb, shade: 155 },
|
||||
{ value: (sample) => sample.javaMb, shade: 80 },
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, memorySeries);
|
||||
|
||||
const cpuPeak = Math.max(...this.samples.map((sample) => sample.cpuPercent), 100);
|
||||
const cpuScale = roundUp(cpuPeak * 1.08, 25);
|
||||
@@ -119,26 +118,16 @@ export class ResourceUsageLayer implements Layer {
|
||||
private takeSample(): void {
|
||||
if (this.removed) return;
|
||||
try {
|
||||
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
|
||||
const atMs = Number(raw[0]);
|
||||
const cpuMs = Number(raw[1]);
|
||||
const elapsedDelta = atMs - this.previousElapsedMs;
|
||||
const cpuDelta = cpuMs - this.previousCpuMs;
|
||||
const sample = sampleResourceUsage();
|
||||
const elapsedDelta = sample.atMs - this.previousElapsedMs;
|
||||
const cpuDelta = sample.cpuMs - this.previousCpuMs;
|
||||
const cpuPercent = this.previousElapsedMs > 0 && elapsedDelta > 0
|
||||
? Math.max(0, cpuDelta / elapsedDelta * 100)
|
||||
: 0;
|
||||
this.previousElapsedMs = atMs;
|
||||
this.previousCpuMs = cpuMs;
|
||||
this.samples.push({
|
||||
atMs,
|
||||
cpuPercent,
|
||||
rssMb: kbToMb(Number(raw[2])),
|
||||
nativeMb: kbToMb(Number(raw[3])),
|
||||
javaMb: kbToMb(Number(raw[4])),
|
||||
javaCommittedMb: kbToMb(Number(raw[5])),
|
||||
threads: Number(raw[6]),
|
||||
});
|
||||
const cutoff = atMs - HISTORY_MS;
|
||||
this.previousElapsedMs = sample.atMs;
|
||||
this.previousCpuMs = sample.cpuMs;
|
||||
this.samples.push({ ...sample, cpuPercent });
|
||||
const cutoff = sample.atMs - HISTORY_MS;
|
||||
while (this.samples.length > 1 && this.samples[0]!.atMs < cutoff) this.samples.shift();
|
||||
while (this.samples.length > MAX_SAMPLES) this.samples.shift();
|
||||
this.sampleError = "";
|
||||
@@ -153,7 +142,7 @@ export class ResourceUsageLayer implements Layer {
|
||||
|
||||
function drawChart(
|
||||
image: GrayImage,
|
||||
samples: readonly ResourceSample[],
|
||||
samples: readonly ChartSample[],
|
||||
nowMs: number,
|
||||
bounds: { x: number; y: number; width: number; height: number },
|
||||
scale: number,
|
||||
@@ -163,7 +152,8 @@ function drawChart(
|
||||
const midY = bounds.y + Math.round((bounds.height - 1) / 2);
|
||||
image.drawLine(bounds.x + 1, midY, bounds.x + bounds.width - 2, midY, 24);
|
||||
const startMs = nowMs - HISTORY_MS;
|
||||
for (const item of series) {
|
||||
// Back to front, so the headline series is drawn on top where lines cross.
|
||||
for (const item of [...series].reverse()) {
|
||||
let previous: { x: number; y: number } | null = null;
|
||||
for (const sample of samples) {
|
||||
if (sample.atMs < startMs) continue;
|
||||
@@ -177,36 +167,30 @@ function drawChart(
|
||||
}
|
||||
}
|
||||
|
||||
/** Right-aligned legend of line swatches and labels; returns its left edge. */
|
||||
function drawLegend(
|
||||
image: GrayImage,
|
||||
font: ReturnType<typeof getDefaultSmallFont>,
|
||||
width: number,
|
||||
y: number,
|
||||
entries: ReadonlyArray<readonly [string, number]>,
|
||||
): void {
|
||||
): number {
|
||||
const totalWidth = entries.reduce((sum, [label]) => sum + font.measureText(label) + 14, 0);
|
||||
let x = width - 16 - totalWidth;
|
||||
const left = width - 16 - totalWidth;
|
||||
let x = left;
|
||||
for (const [label, shade] of entries) {
|
||||
image.drawLine(x, y + 6, x + 8, y + 6, shade);
|
||||
x += 12;
|
||||
image.drawText(font, x, y, label, shade);
|
||||
x += font.measureText(label) + 2;
|
||||
}
|
||||
}
|
||||
|
||||
function kbToMb(kb: number): number {
|
||||
return Math.max(0, kb) / 1024;
|
||||
return left;
|
||||
}
|
||||
|
||||
function roundUp(value: number, step: number): number {
|
||||
return Math.max(step, Math.ceil(value / step) * step);
|
||||
}
|
||||
|
||||
function formatMb(value: number): string {
|
||||
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
|
||||
return `${Math.round(value)}M`;
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return `${value < 10 ? value.toFixed(1) : Math.round(value)}%`;
|
||||
}
|
||||
@@ -176,6 +176,21 @@ export function readEvenHubPackageManifestBytes(bytes: Uint8Array): EvenHubManif
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an installed EHPK that is all zeros, keeping its registry entry and
|
||||
* icon so the store treats it as missing and offers Reinstall. iOS builds
|
||||
* before the binary-http.ios.ts copy fix could store a large download that
|
||||
* way. Returns whether the package was discarded.
|
||||
*/
|
||||
export function discardZeroedEvenHubPackage(packageId: string): boolean {
|
||||
const path = installedEvenHubPackagePath(packageId);
|
||||
const bytes = readBinaryFile(path);
|
||||
if (!bytes || bytes.length === 0) return false;
|
||||
for (let i = 0; i < bytes.length; i++) if (bytes[i] !== 0) return false;
|
||||
deletePathRecursively(path);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Remove the stored EHPK, unpacked runtime files, and registry entry. */
|
||||
export function uninstallEvenHubPackage(packageId: string): boolean {
|
||||
const apps = getInstalledEvenHubApps();
|
||||
|
||||
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
|
||||
deliverAudioPcm(pcm: Uint8Array): void;
|
||||
};
|
||||
|
||||
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
|
||||
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
* flows (foreground + screen on + no assistant modal).
|
||||
*/
|
||||
private audioControl(data: Record<string, unknown>): boolean {
|
||||
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
|
||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||
return false;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { type GrayImage, type UiFont } from "../../../graphics/image";
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../../native/location-permissions";
|
||||
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
|
||||
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
||||
import { weatherBridge } from "../../../native/weather";
|
||||
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
||||
@@ -37,7 +38,6 @@ import { glanceFont } from "../glance-font";
|
||||
import { type GlanceWidget } from "../widget";
|
||||
|
||||
const PAD = 8;
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const NOTIFICATION_ICON_GAP = 4;
|
||||
const BATTERY_ITEM_GAP = 12;
|
||||
const BATTERY_LABEL_VALUE = 150;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||
|
||||
@@ -245,7 +245,7 @@ export function askConversations(
|
||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
return streamLocalQwen({
|
||||
return streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { useMicControlSetting } from "../../ui/dashboard-settings";
|
||||
import { toJavaBytes } from "../../native/cloud-stt";
|
||||
import { toUint8Array } from "../../util/array-util";
|
||||
import { addMicStatusListener, micControlSession, sendMicControlRecord } from "../../native/mic-control-link";
|
||||
import {
|
||||
MIC_LEASE_MS,
|
||||
decodeMicStatus,
|
||||
@@ -14,8 +13,6 @@ import {
|
||||
type MicStatus,
|
||||
} from "./mic-protocol";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Host-side controller for the CFW mic_control array: sends the same config
|
||||
* to both temples (each temple is its own endpoint and its own 2-mic array),
|
||||
@@ -30,15 +27,6 @@ const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
|
||||
|
||||
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the "Use microphone control" developer setting is on. With the
|
||||
* setting off, Faceclaw behaves exactly as it would on firmware without the
|
||||
@@ -71,9 +59,9 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
// The Java listener proxy must stay referenced or it gets GC'd.
|
||||
private statusProxy: any = null;
|
||||
private proxyCommunicator: any = null;
|
||||
// Holds the native listener proxy, which must stay referenced or it gets GC'd.
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private statusSession: unknown = null;
|
||||
private leaseTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private armed = false;
|
||||
|
||||
@@ -158,39 +146,28 @@ class MicArrayController {
|
||||
rightTemple = true,
|
||||
leftTemple = true,
|
||||
): boolean {
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return false;
|
||||
const session = micControlSession();
|
||||
if (!session) return false;
|
||||
try {
|
||||
this.ensureStatusProxy(communicator);
|
||||
const record = encodeMicControl(op, config);
|
||||
// False when the session wasn't ready and nothing was queued.
|
||||
return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
this.ensureStatusListener(session);
|
||||
return sendMicControlRecord(session, encodeMicControl(op, config), label, rightTemple, leftTemple);
|
||||
} catch (error) {
|
||||
console.warn(`mic control ${label} failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private ensureStatusProxy(communicator: any): void {
|
||||
if (this.statusProxy && this.proxyCommunicator === communicator) return;
|
||||
if (this.statusProxy && this.proxyCommunicator) {
|
||||
try {
|
||||
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
}
|
||||
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
||||
onMicStatus: (body: any, arm: string) => {
|
||||
const status = decodeMicStatus(toUint8Array(body));
|
||||
if (!status) return;
|
||||
const side: MicSide = String(arm) === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
},
|
||||
private ensureStatusListener(session: unknown): void {
|
||||
if (this.unsubscribeStatus && this.statusSession === session) return;
|
||||
this.unsubscribeStatus?.();
|
||||
this.statusSession = session;
|
||||
this.unsubscribeStatus = addMicStatusListener(session, (body, arm) => {
|
||||
const status = decodeMicStatus(body);
|
||||
if (!status) return;
|
||||
const side: MicSide = arm === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
});
|
||||
this.proxyCommunicator = communicator;
|
||||
communicator.addMicStatusListener(this.statusProxy);
|
||||
}
|
||||
|
||||
private syncLeaseTimer(): void {
|
||||
|
||||
@@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment";
|
||||
import { generateSessionInsights } from "./speaker-insights";
|
||||
import { speakerRegistry, type SpeakerMatch } from "./speakers";
|
||||
import { postAmbientCard } from "../../ui/shell/ambient-cards";
|
||||
import { pcmLevel, rmsToLevel } from "../../util/audio-level";
|
||||
import { formatRelativeTime } from "../../util/date-util";
|
||||
import { deviceLanguage, identifyLanguage, translateText } from "./translate";
|
||||
|
||||
@@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig {
|
||||
return { ...config, channelMask: 0b11 };
|
||||
}
|
||||
|
||||
function pcmLevel(pcm: Uint8Array): number {
|
||||
let sumSquares = 0;
|
||||
const count = pcm.length / 2;
|
||||
if (count === 0) return 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
if (value >= 0x8000) value -= 0x10000;
|
||||
sumSquares += value * value;
|
||||
}
|
||||
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
|
||||
}
|
||||
|
||||
function floatLevel(samples: Float32Array): number {
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
|
||||
return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length)));
|
||||
}
|
||||
|
||||
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
|
||||
function rmsToLevel(rms: number): number {
|
||||
if (rms <= 0) return 0;
|
||||
const db = 20 * Math.log10(rms);
|
||||
return Math.max(0, Math.min(1, (db + 60) / 60));
|
||||
}
|
||||
|
||||
function floatToPcm16(samples: Float32Array): Uint8Array {
|
||||
const out = new Uint8Array(samples.length * 2);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { NameInference } from "./name-guess";
|
||||
|
||||
/**
|
||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import { speakerRegistry } from "./speakers";
|
||||
|
||||
declare const com: any;
|
||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
+53
-14
@@ -8,8 +8,10 @@ import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { mediaControllerBridge, type MediaControllerState, type MediaQueueItem } from "../../native/media-controller";
|
||||
import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser";
|
||||
import { spotifyService } from "../../native/spotify";
|
||||
import { MediaBrowseLayer } from "./media-browse";
|
||||
import { MusicSettingsLayer } from "./music-settings";
|
||||
import { spotifyMenuItem } from "./spotify-menu";
|
||||
import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
|
||||
@@ -33,6 +35,8 @@ const QUEUE_X = 204;
|
||||
/** Horizontal inset of row text from its selection box. */
|
||||
const LIST_TEXT_INSET = 6;
|
||||
const PLAYLIST_ACTION_INDEX = 1;
|
||||
/** Minimum room for the dim artist after a queue row's title; less and it is left off. */
|
||||
const QUEUE_SUBTITLE_MIN_WIDTH = 40;
|
||||
|
||||
type MusicAction =
|
||||
| { kind: "play-pause"; label: string; enabled: boolean }
|
||||
@@ -40,14 +44,19 @@ type MusicAction =
|
||||
| { kind: "next"; label: string; enabled: boolean }
|
||||
| { kind: "volume"; label: string; enabled: boolean }
|
||||
| { kind: "playlist"; label: string; enabled: boolean }
|
||||
| { kind: "browse"; label: string; enabled: boolean };
|
||||
| { kind: "browse"; label: string; enabled: boolean }
|
||||
/** A player-specific action from the state's customActions (Spotify's shuffle / repeat / like). */
|
||||
| { kind: "custom"; label: string; enabled: boolean; action: string };
|
||||
|
||||
type FocusColumn = "actions" | "playlist";
|
||||
|
||||
/**
|
||||
* Music controller app: metadata + album art + transport controls for the
|
||||
* active Android media session (any player that publishes one) or, on iOS,
|
||||
* the Apple Music system player, plus the player's queue when it exposes one
|
||||
* the Apple Music system player; on both, Spotify through its Web API once
|
||||
* signed in from the window menu (optional on Android, where it adds
|
||||
* library browsing, shuffle / repeat / like and Spotify on other devices),
|
||||
* plus the player's queue when it exposes one
|
||||
* (scroll to a track, click to jump). With no session, the front page lists
|
||||
* the apps whose library can be browsed (click one to open its
|
||||
* MediaBrowseLayer).
|
||||
@@ -110,7 +119,7 @@ class MusicAppLayer implements Layer {
|
||||
const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh);
|
||||
this.libraryAppsFresh = true;
|
||||
this.libraryMenu.setItems(apps);
|
||||
image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180);
|
||||
image.drawText(font, 24, 16, global.isIOS ? media.status || "Nothing is playing." : "No active media session.", 180);
|
||||
const prompt = global.isIOS
|
||||
? "Browse your library, or start playback on the phone."
|
||||
: apps.length
|
||||
@@ -121,6 +130,11 @@ class MusicAppLayer implements Layer {
|
||||
image.drawText(font, 24, y, line, 150);
|
||||
y += lineStep(font);
|
||||
}
|
||||
// Why the last command failed (e.g. playing from the Spotify library with no device open).
|
||||
for (const line of media.notice ? wrapText(font, media.notice, width - 48) : []) {
|
||||
image.drawText(font, 24, y, line, 200);
|
||||
y += lineStep(font);
|
||||
}
|
||||
if (apps.length) {
|
||||
const listTop = y + 6;
|
||||
const listHeight = height - listTop - LIST_BOTTOM_MARGIN;
|
||||
@@ -156,12 +170,13 @@ class MusicAppLayer implements Layer {
|
||||
if (media.album) {
|
||||
image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150);
|
||||
}
|
||||
// The player line, or for a few seconds why a command failed.
|
||||
image.drawText(
|
||||
font,
|
||||
META_X,
|
||||
metaTop + 4 + (titleSlots + 2) * metaStep,
|
||||
truncateText(font, media.appName || media.packageName, metaWidth),
|
||||
110,
|
||||
truncateText(font, media.notice || media.appName || media.packageName, metaWidth),
|
||||
media.notice ? 200 : 110,
|
||||
);
|
||||
this.drawProgress(image, media, metaWidth);
|
||||
|
||||
@@ -212,7 +227,16 @@ class MusicAppLayer implements Layer {
|
||||
private drawQueueItem({ image, item, x, y, width, height, selected }: MenuDrawArgs<MediaQueueItem>): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const label = `${item.active ? "> " : " "}${item.title || "(untitled)"}`;
|
||||
image.drawText(font, x + LIST_TEXT_INSET, centeredTextY(font, y, height), truncateText(font, label, width - 2 * LIST_TEXT_INSET), selected ? 255 : 200);
|
||||
const textWidth = width - 2 * LIST_TEXT_INSET;
|
||||
const textY = centeredTextY(font, y, height);
|
||||
const title = truncateText(font, label, textWidth);
|
||||
image.drawText(font, x + LIST_TEXT_INSET, textY, title, selected ? 255 : 200);
|
||||
// The artist, dimmer, in whatever room the title leaves.
|
||||
const subtitleX = font.measureText(`${label} `);
|
||||
if (item.subtitle && title === label && textWidth - subtitleX >= QUEUE_SUBTITLE_MIN_WIDTH) {
|
||||
image.drawText(font, x + LIST_TEXT_INSET + subtitleX, textY,
|
||||
truncateText(font, item.subtitle, textWidth - subtitleX), selected ? 150 : 110);
|
||||
}
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
@@ -278,6 +302,7 @@ class MusicAppLayer implements Layer {
|
||||
else if (action.kind === "previous") await mediaControllerBridge.skipPrevious();
|
||||
else if (action.kind === "next") await mediaControllerBridge.skipNext();
|
||||
else if (action.kind === "browse") this.openBrowse(ctx);
|
||||
else if (action.kind === "custom") await mediaControllerBridge.sendCustomAction?.(action.action);
|
||||
else if (action.kind === "volume") {
|
||||
ctx.stack.push(new VolumeModalLayer(mediaControllerBridge.getMediaVolumePercent()));
|
||||
} else if (action.kind === "playlist") {
|
||||
@@ -342,6 +367,12 @@ class MusicAppLayer implements Layer {
|
||||
{ kind: "volume", label: volume >= 0 ? `Volume (${volume})` : "Volume", enabled: volume >= 0 },
|
||||
{ kind: "next", label: "Next track", enabled: media.canSkipNext },
|
||||
{ kind: "previous", label: "Previous track", enabled: media.canSkipPrevious },
|
||||
...(mediaControllerBridge.sendCustomAction ? media.customActions ?? [] : []).map((custom): MusicAction => ({
|
||||
kind: "custom",
|
||||
label: custom.name,
|
||||
enabled: true,
|
||||
action: custom.action,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -496,14 +527,22 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin
|
||||
iconLetter: "M",
|
||||
icon: "music",
|
||||
closeable: true,
|
||||
// Music settings choose among Android media apps; iOS has only Apple Music.
|
||||
menuItems: () => global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}],
|
||||
// Music settings choose among Android media apps (iOS has Apple Music
|
||||
// alone); Spotify's sign-in is optional on Android, where its media
|
||||
// session works without it.
|
||||
menuItems: () => {
|
||||
const spotify = spotifyService();
|
||||
return [
|
||||
...(global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx: LayerContext) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}]),
|
||||
...(spotify ? [spotifyMenuItem(spotify)] : []),
|
||||
];
|
||||
},
|
||||
actions: options.actions,
|
||||
baseLayer: new MusicRootLayer(musicLayer),
|
||||
submitFrame: options.submitFrame,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { GrayImage } from "../../graphics/image";
|
||||
import { isValidSpotifyClientId, SPOTIFY_DASHBOARD_URL, SPOTIFY_REDIRECT_URI } from "../../native/spotify-auth";
|
||||
import type { SpotifyService } from "../../native/spotify-service";
|
||||
import { spotifyClientIdSetting, textSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { submenuItem, TextPageLayer, type MenuItem } from "../../ui/menu";
|
||||
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||
import { TextViewerLayer } from "../files/text-viewer";
|
||||
|
||||
const SETUP_INTRO = global.isIOS
|
||||
? "Faceclaw controls Spotify through Spotify's Web API, which needs a Spotify developer app of your own. Creating one is free; Spotify requires its owner to have Premium, and controlling playback needs Premium too."
|
||||
: "Faceclaw already shows and controls Spotify on this phone, like any music app. Signing in to Spotify's Web API is optional and adds browsing your Spotify library, playing from Spotify by voice, shuffle / repeat / like, and Spotify playing on your other devices. It needs a Spotify developer app of your own: creating one is free, Spotify requires its owner to have Premium, and controlling playback through it needs Premium too.";
|
||||
|
||||
const SETUP_HELP = [
|
||||
SETUP_INTRO,
|
||||
"",
|
||||
`1. On a computer, open ${SPOTIFY_DASHBOARD_URL.replace("https://", "")} and sign in with your Spotify account.`,
|
||||
"2. Create an app. Any name and description will do.",
|
||||
`3. Add this Redirect URI exactly: ${SPOTIFY_REDIRECT_URI}`,
|
||||
"4. Under \"Which API/SDKs are you planning to use?\", tick Web API, then save.",
|
||||
"5. Open the app's settings, copy its Client ID, and enter it as Client ID here.",
|
||||
"6. Choose Sign in, then finish on your phone: Faceclaw opens Spotify's sign-in page there (once Faceclaw is open on the phone, if it was in the background).",
|
||||
"",
|
||||
"Spotify plays on whichever of your devices is open (this phone, a computer, a speaker). To start music when nothing is playing, Spotify must be open on one of them.",
|
||||
"",
|
||||
"Others can sign in with your client ID once you add their Spotify email under User Management in the dashboard (up to five people).",
|
||||
].join("\n");
|
||||
|
||||
/** The Music app menu's Spotify row: sign in, sign out, client ID, setup help. */
|
||||
export function spotifyMenuItem(service: SpotifyService): MenuItem {
|
||||
return submenuItem("Spotify", (ctx) => ctx.stack.push(new SpotifyMenuLayer(service)), {
|
||||
description: global.isIOS
|
||||
? "Sign in to Spotify to show and control it from the glasses."
|
||||
: "Optional: sign in to Spotify to browse its library and play from it by voice.",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows rebuilt every paint and input from the account and sign-in progress,
|
||||
* which change asynchronously. A WindowMenuLayer, so the shell's system menu
|
||||
* closes it like the window menu it opens from.
|
||||
*/
|
||||
class SpotifyMenuLayer extends WindowMenuLayer {
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
constructor(private readonly service: SpotifyService) {
|
||||
super("Spotify", []);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
this.unsubscribe ??= this.service.onChange(() => ctx.actions.requestRender());
|
||||
this.setItems(this.rows());
|
||||
return super.paint(ctx, paintBelow);
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
this.setItems(this.rows());
|
||||
await super.handleInput(event, ctx);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
private rows(): MenuItem[] {
|
||||
const service = this.service;
|
||||
const account = service.account();
|
||||
const status = service.signInStatus();
|
||||
const help: MenuItem = {
|
||||
label: "Setup help",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextViewerLayer(SETUP_HELP, "Spotify setup")),
|
||||
};
|
||||
if (account) {
|
||||
return [
|
||||
info(`Signed in as ${account.displayName || account.userId}`),
|
||||
...(account.product && account.product !== "premium" ? [info("Playback control needs Premium")] : []),
|
||||
{ label: "Sign out", onSelect: () => service.signOut() },
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
];
|
||||
}
|
||||
if (status.phase === "waiting") {
|
||||
return [
|
||||
info("Finish signing in on the phone"),
|
||||
{ label: "Cancel sign-in", onSelect: () => service.cancelSignIn() },
|
||||
help,
|
||||
];
|
||||
}
|
||||
const failure: MenuItem[] = status.error
|
||||
? [{
|
||||
label: "Sign-in failed (details)",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextPageLayer("Spotify sign-in failed", status.error)),
|
||||
}]
|
||||
: [];
|
||||
return [
|
||||
{
|
||||
label: "Sign in",
|
||||
description: "Opens Spotify's sign-in page on the phone.",
|
||||
disabled: () => !isValidSpotifyClientId(spotifyClientIdSetting.get()),
|
||||
onSelect: () => void service.signIn(),
|
||||
},
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
...failure,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
function info(label: string): MenuItem {
|
||||
return { label, disabled: true, onSelect: () => {} };
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import { WindowMenu, WindowMenuLayer } from "../../ui/window-menu";
|
||||
import type { LayerContext } from "../../ui/layers";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { onSettingsStoreChanged } from "../../native/settings-store";
|
||||
import { formatClockTime } from "../../ui/clock-format";
|
||||
import { openUrlOnPhone } from "../../native/open-url";
|
||||
import {
|
||||
navigateDestinationAddressDraftSetting,
|
||||
@@ -35,6 +36,9 @@ import {
|
||||
navigateRememberRecentSetting,
|
||||
navigateDisplayLayoutSetting,
|
||||
mapboxApiKeySetting,
|
||||
distanceUnitSetting,
|
||||
DISTANCE_UNIT_SETTING_KEY,
|
||||
TIME_FORMAT_SETTING_KEY,
|
||||
enumSettingMenuItem,
|
||||
type ConfigSettingString,
|
||||
} from "../../ui/dashboard-settings";
|
||||
@@ -266,8 +270,15 @@ function post(message: WorkerAppReply): void {
|
||||
// the edit screen live.
|
||||
onSettingsStoreChanged((key) => {
|
||||
// The token key repaints too: the phone editor writes it live during
|
||||
// Edit token, and the idle page changes shape once one is set.
|
||||
if (key.startsWith("navigate.") || key === MAPBOX_TOKEN_SETTING_KEY) render();
|
||||
// Edit token, and the idle page changes shape once one is set. So do the
|
||||
// distance unit and time format (Settings > Customization), for the
|
||||
// guidance panel.
|
||||
if (
|
||||
key.startsWith("navigate.") ||
|
||||
key === MAPBOX_TOKEN_SETTING_KEY ||
|
||||
key === DISTANCE_UNIT_SETTING_KEY ||
|
||||
key === TIME_FORMAT_SETTING_KEY
|
||||
) render();
|
||||
});
|
||||
|
||||
// The host queues messages until this arrives: posts to a worker whose bundle
|
||||
@@ -1569,15 +1580,33 @@ function drawFooter(image: GrayImage, text: string, x = 16): void {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Formatting (imperial; matches the rest of the UI)
|
||||
// Formatting (distances and clock times follow Settings > Customization)
|
||||
|
||||
function metricDistances(): boolean {
|
||||
return distanceUnitSetting.get() === "kilometers";
|
||||
}
|
||||
|
||||
/** Under a kilometer: 10 m steps up to 100 m, then 50 m steps. */
|
||||
function formatMeters(meters: number): string {
|
||||
const step = meters < 100 ? 10 : 50;
|
||||
return `${Math.max(0, Math.round(meters / step) * step)} m`;
|
||||
}
|
||||
|
||||
function formatDistanceShort(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
return meters < 950 ? formatMeters(meters) : `${(meters / 1000).toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 900) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
return `${(feet / 5280).toFixed(1)} mi`;
|
||||
}
|
||||
|
||||
function formatDistance(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
if (meters < 950) return formatMeters(meters);
|
||||
const km = meters / 1000;
|
||||
return km >= 10 ? `${Math.round(km)} km` : `${km.toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 1000) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
const miles = feet / 5280;
|
||||
@@ -1597,14 +1626,9 @@ function formatDurationLong(seconds: number): string {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
}
|
||||
|
||||
/** "14:05" / "2:05 PM", as the top-bar clock shows it. */
|
||||
function formatClock(timestampMs: number): string {
|
||||
const date = new Date(timestampMs);
|
||||
let hours = date.getHours();
|
||||
const minutes = String(date.getMinutes()).padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
return formatClockTime(new Date(timestampMs));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer {
|
||||
private detail = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private readonly meter = new MicLevelMeter();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
@@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer {
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
@@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer {
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
@@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { TranslateLayer } from "./translate";
|
||||
import { resolveTranslationModel } from "./translate-llm";
|
||||
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
|
||||
import {
|
||||
resolveTranslateLanguage,
|
||||
resolveTranslateProvider,
|
||||
translateLanguageSetting,
|
||||
translateProviderSetting,
|
||||
translateShowOriginalSetting,
|
||||
} from "./translate-settings";
|
||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
@@ -22,8 +28,8 @@ const TRAY_ICON_ID = "translate";
|
||||
const NO_PROVIDER_PROBLEM =
|
||||
"Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice).";
|
||||
const NO_MODEL_PROBLEM = global.isIOS
|
||||
? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)."
|
||||
: "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant).";
|
||||
? "Translate needs a language model: configure a cloud or OpenAI-compatible model in Settings > Assistant."
|
||||
: "Translate needs a language model: configure one in Settings > Assistant, or download the on-phone model.";
|
||||
|
||||
export type TranslateAppOptions = InProcessAppOptions & {
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
@@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & {
|
||||
* provider (Translate settings), which a shared capture switches to.
|
||||
*/
|
||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||
// Checks the setup and (re)starts capture with the chosen provider: at
|
||||
// launch, when the provider setting changes, and on a tap after a problem.
|
||||
// Checks the setup and (re)starts capture with the chosen provider and
|
||||
// language: at launch, when either setting changes, and on a tap after a
|
||||
// problem.
|
||||
const startCapture = () => {
|
||||
const provider = resolveTranslateProvider();
|
||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||
@@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
return;
|
||||
}
|
||||
options.startContinuousVoiceCapture({ multilingualProvider: provider });
|
||||
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
};
|
||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
||||
@@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Translate settings", [
|
||||
enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }),
|
||||
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
|
||||
toggleSettingMenuItem(translateShowOriginalSetting),
|
||||
]);
|
||||
@@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
||||
startCapture();
|
||||
return app;
|
||||
}
|
||||
@@ -4,11 +4,15 @@
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import {
|
||||
anthropicApiKeySetting, assistantModelSetting, assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts";
|
||||
import { type TranslationTurn } from "./translation-model";
|
||||
|
||||
@@ -23,12 +27,17 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null {
|
||||
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
}
|
||||
|
||||
export type TranslationRequest = {
|
||||
speech: string;
|
||||
context: TranslationTurn[];
|
||||
/** English name of the spoken language when the user set one, else "". */
|
||||
language: string;
|
||||
onProgress: (textSoFar: string) => void;
|
||||
onDone: (translation: string) => void;
|
||||
onError: (message: string) => void;
|
||||
@@ -41,7 +50,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
system: TRANSLATE_SYSTEM_PROMPT,
|
||||
effort: model.effort,
|
||||
maxTokens: 1024,
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }],
|
||||
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
|
||||
onDone: (result) => {
|
||||
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
|
||||
@@ -52,8 +61,10 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
switch (model.provider) {
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "openai-compatible":
|
||||
return streamOpenAiCompatibleChat({ ...options, baseUrl: model.baseUrl ?? "" });
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
return streamLocalModel(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
|
||||
@@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>
|
||||
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Spoken languages Translate can be told to expect, by ISO 639-1 code (the
|
||||
* form every multilingual provider accepts), in picker order.
|
||||
*/
|
||||
const SPOKEN_LANGUAGES = {
|
||||
ar: "Arabic",
|
||||
zh: "Chinese",
|
||||
nl: "Dutch",
|
||||
fr: "French",
|
||||
de: "German",
|
||||
el: "Greek",
|
||||
he: "Hebrew",
|
||||
hi: "Hindi",
|
||||
id: "Indonesian",
|
||||
it: "Italian",
|
||||
ja: "Japanese",
|
||||
ko: "Korean",
|
||||
pl: "Polish",
|
||||
pt: "Portuguese",
|
||||
ru: "Russian",
|
||||
es: "Spanish",
|
||||
sv: "Swedish",
|
||||
th: "Thai",
|
||||
tr: "Turkish",
|
||||
uk: "Ukrainian",
|
||||
vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES;
|
||||
|
||||
export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({
|
||||
id: "translate-language",
|
||||
label: "Spoken language",
|
||||
storageKey: "translate.language",
|
||||
defaultValue: "auto",
|
||||
values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])],
|
||||
formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]),
|
||||
description:
|
||||
"The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.",
|
||||
});
|
||||
|
||||
/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */
|
||||
export function resolveTranslateLanguage(): SpokenLanguage | "" {
|
||||
const value = translateLanguageSetting.get();
|
||||
return value === "auto" ? "" : value;
|
||||
}
|
||||
|
||||
/** The English name of a spoken language, for the translation prompt. */
|
||||
export function spokenLanguageName(language: SpokenLanguage): string {
|
||||
return SPOKEN_LANGUAGES[language];
|
||||
}
|
||||
|
||||
export const translateShowOriginalSetting = new ConfigSettingBoolean({
|
||||
id: "translate-show-original",
|
||||
label: "Show original",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
||||
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
||||
import { translateShowOriginalSetting } from "./translate-settings";
|
||||
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
||||
|
||||
const NO_MODEL_NOTICE = "No language model is set up.";
|
||||
|
||||
/**
|
||||
* Live translation view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranslateAppWindow); this layer feeds finalized speech to the
|
||||
* assistant's model one stretch at a time, shows the English as it streams
|
||||
* in, and saves the session to Downloads from the app menu.
|
||||
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
||||
* model one block at a time (provisionally while a block is still being
|
||||
* spoken), shows the English as it streams in, and saves the session to
|
||||
* Downloads from the app menu.
|
||||
*/
|
||||
export class TranslateLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -30,7 +32,8 @@ export class TranslateLayer implements Layer {
|
||||
private problem: string | null = null;
|
||||
private readonly model = new TranslationModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private activeJob: TranslationJob | null = null;
|
||||
private readonly meter = new MicLevelMeter();
|
||||
// The request in flight (the model tracks which job it is).
|
||||
private activeHandle: LlmStreamHandle | null = null;
|
||||
private requestRender: () => void = () => {};
|
||||
private removed = false;
|
||||
@@ -40,10 +43,16 @@ export class TranslateLayer implements Layer {
|
||||
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
||||
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.requestRender = requestRender;
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.model.accept(event);
|
||||
if (this.model.accept(event)) {
|
||||
// The request in flight translates text the transcript has replaced.
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
}
|
||||
this.pump();
|
||||
requestRender();
|
||||
});
|
||||
@@ -68,9 +77,9 @@ export class TranslateLayer implements Layer {
|
||||
const measure = (line: string) => font.measureText(line);
|
||||
const maxWidth = width - 64;
|
||||
|
||||
image.drawText(font, 24, 20, "Translate", 200);
|
||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
||||
image.drawText(font, 24, 40, this.saveNotice || status, 110);
|
||||
const meter = this.listening && !this.problem ? this.meter : null;
|
||||
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
|
||||
|
||||
let lines: TranslationLine[];
|
||||
if (this.problem) {
|
||||
@@ -84,7 +93,6 @@ export class TranslateLayer implements Layer {
|
||||
}
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
||||
@@ -119,7 +127,7 @@ export class TranslateLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.activeJob = null;
|
||||
this.meter.stop();
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
@@ -139,7 +147,10 @@ export class TranslateLayer implements Layer {
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
/** Start translating the next waiting stretch of speech, unless one is in flight. */
|
||||
/**
|
||||
* Start the next translation request (a final block, else the live block's
|
||||
* provisional one), unless one is in flight.
|
||||
*/
|
||||
private pump(): void {
|
||||
if (this.removed) return;
|
||||
const job = this.model.nextJob();
|
||||
@@ -147,36 +158,34 @@ export class TranslateLayer implements Layer {
|
||||
const llm = resolveTranslationModel();
|
||||
if (!llm) {
|
||||
// The model went away mid-session (a key was cleared): keep the original.
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = NO_MODEL_NOTICE;
|
||||
this.pump();
|
||||
return;
|
||||
}
|
||||
this.activeJob = job;
|
||||
const language = resolveTranslateLanguage();
|
||||
const handle = streamTranslation(llm, {
|
||||
speech: job.segment.source,
|
||||
speech: job.source,
|
||||
context: job.context,
|
||||
language: language ? spokenLanguageName(language) : "",
|
||||
onProgress: (textSoFar) => {
|
||||
if (this.activeJob !== job) return;
|
||||
this.model.progress(job.segment, textSoFar);
|
||||
this.requestRender();
|
||||
if (this.model.progress(job, textSoFar)) this.requestRender();
|
||||
},
|
||||
onDone: (translation) => this.settle(job, () => {
|
||||
this.model.complete(job.segment, translation);
|
||||
this.model.complete(job, translation);
|
||||
this.translationError = "";
|
||||
}),
|
||||
onError: (message) => this.settle(job, () => {
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = `Translation failed: ${message}`;
|
||||
}),
|
||||
});
|
||||
// A provider that fails synchronously has already settled (and moved on).
|
||||
if (this.activeJob === job) this.activeHandle = handle;
|
||||
if (this.model.isActive(job)) this.activeHandle = handle;
|
||||
}
|
||||
|
||||
private settle(job: TranslationJob, apply: () => void): void {
|
||||
if (this.activeJob !== job) return;
|
||||
this.activeJob = null;
|
||||
if (!this.model.isActive(job)) return;
|
||||
this.activeHandle = null;
|
||||
apply();
|
||||
this.pump();
|
||||
|
||||
@@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
|
||||
type TranscriptEvent = { text: string; isFinal: boolean };
|
||||
|
||||
export type TranslationState = "pending" | "translating" | "done" | "failed";
|
||||
|
||||
/** One stretch of finalized speech and its English translation. */
|
||||
/**
|
||||
* One block of speech (up to a pause) and its English translation. The last
|
||||
* block may still be live, its transcript revising; it is translated
|
||||
* provisionally as it grows, then again once final if its text changed.
|
||||
*/
|
||||
export type TranslationSegment = {
|
||||
source: string;
|
||||
/** The translation so far (streamed while translating). */
|
||||
/** Committed by the transcriber; a live block's source still revises. */
|
||||
final: boolean;
|
||||
/** The latest finished translation, of `translatedSource`. */
|
||||
translation: string;
|
||||
state: TranslationState;
|
||||
/** What `translation` translates: null until one finishes, behind `source` while provisional. */
|
||||
translatedSource: string | null;
|
||||
/** The text streamed so far by the request in flight for this block, else null. */
|
||||
streaming: string | null;
|
||||
/** A source whose translation failed; not retried until the source changes. */
|
||||
failedSource: string | null;
|
||||
};
|
||||
|
||||
export type TranslationTurn = { source: string; translation: string };
|
||||
|
||||
/** A segment handed to the model, with the finished turns before it. */
|
||||
/** A request: a block's source as sent, with the finished turns before it. */
|
||||
export type TranslationJob = {
|
||||
segment: TranslationSegment;
|
||||
/** The source sent (a live block's moves on while the request is in flight). */
|
||||
source: string;
|
||||
/** Earlier translated turns, oldest first. */
|
||||
context: TranslationTurn[];
|
||||
};
|
||||
@@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number };
|
||||
// over between requests; bounded so a long session's requests stay small.
|
||||
const CONTEXT_TURNS = 4;
|
||||
const CONTEXT_CHARS = 800;
|
||||
// A live block is translated again once its text has changed length by this
|
||||
// much since the last request, so a trickle of revisions (punctuation, one
|
||||
// corrected word) doesn't spend a request each. A final is always redone.
|
||||
const PROVISIONAL_MIN_CHANGE = 3;
|
||||
|
||||
const TRANSLATION_GRAY = 230;
|
||||
const PROVISIONAL_GRAY = 170;
|
||||
const ORIGINAL_GRAY = 120;
|
||||
const LIVE_GRAY = 90;
|
||||
|
||||
function newSegment(source: string, final: boolean): TranslationSegment {
|
||||
return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null };
|
||||
}
|
||||
|
||||
/** Whether a block's current source still needs a translation request. */
|
||||
function wantsTranslation(segment: TranslationSegment): boolean {
|
||||
const { source, translatedSource } = segment;
|
||||
if (!source || source === translatedSource || source === segment.failedSource) return false;
|
||||
return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Translate app's conversation: finalized speech segments with their
|
||||
* translations, plus the live partial of the utterance in progress.
|
||||
* The Translate app's conversation: blocks of speech with their translations.
|
||||
* Translation is serial (one request at a time, in order, so each can carry
|
||||
* the turns before it as context); finals that arrive while a request is in
|
||||
* flight merge into one pending segment, so a slow model batches the backlog
|
||||
* instead of falling further behind one request per pause.
|
||||
* the turns before it as context): final blocks first, then the live one.
|
||||
* Finals that arrive while a request is in flight merge into one waiting
|
||||
* block, so a slow model batches the backlog instead of falling further
|
||||
* behind one request per pause.
|
||||
*/
|
||||
export class TranslationModel {
|
||||
readonly segments: TranslationSegment[] = [];
|
||||
/** The utterance in progress (replace semantics), not yet final. */
|
||||
live = "";
|
||||
private active: TranslationJob | null = null;
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
/**
|
||||
* Take a transcript event (replace semantics within the live block).
|
||||
* Returns true when it made the request in flight moot (its block was
|
||||
* finalized as other text, or dropped); that request is abandoned here, and
|
||||
* the caller should cancel it.
|
||||
*/
|
||||
accept(event: TranscriptEvent): boolean {
|
||||
const text = event.text.trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
this.live = "";
|
||||
if (!finalText) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
|
||||
else this.segments.push({ source: finalText, translation: "", state: "pending" });
|
||||
const live = last && !last.final ? last : null;
|
||||
if (!event.isFinal) {
|
||||
if (live) live.source = text;
|
||||
else if (text) this.segments.push(newSegment(text, false));
|
||||
return false;
|
||||
}
|
||||
const finalText = text || live?.source || "";
|
||||
if (!live) {
|
||||
if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true));
|
||||
return false;
|
||||
}
|
||||
const moot = this.active?.segment === live && this.active.source !== finalText;
|
||||
if (moot) this.abandonActive();
|
||||
if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) {
|
||||
this.segments.pop();
|
||||
return moot;
|
||||
}
|
||||
live.source = finalText;
|
||||
live.final = true;
|
||||
return moot;
|
||||
}
|
||||
|
||||
/** Take the next segment to translate, or null while one is in flight or none waits. */
|
||||
/** Take the next request to make, or null while one is in flight or nothing waits. */
|
||||
nextJob(): TranslationJob | null {
|
||||
if (this.segments.some((segment) => segment.state === "translating")) return null;
|
||||
const segment = this.segments.find((candidate) => candidate.state === "pending");
|
||||
if (this.active) return null;
|
||||
const segment = this.segments.find((candidate) => wantsTranslation(candidate));
|
||||
if (!segment) return null;
|
||||
segment.state = "translating";
|
||||
return { segment, context: this.contextBefore(segment) };
|
||||
segment.streaming = "";
|
||||
this.active = { segment, source: segment.source, context: this.contextBefore(segment) };
|
||||
return this.active;
|
||||
}
|
||||
|
||||
progress(segment: TranslationSegment, textSoFar: string): void {
|
||||
if (segment.state === "translating") segment.translation = textSoFar;
|
||||
isActive(job: TranslationJob): boolean {
|
||||
return this.active === job;
|
||||
}
|
||||
|
||||
complete(segment: TranslationSegment, translation: string): void {
|
||||
segment.translation = translation;
|
||||
segment.state = "done";
|
||||
/** Streamed text for the request in flight; false when `job` isn't it any more. */
|
||||
progress(job: TranslationJob, textSoFar: string): boolean {
|
||||
if (this.active !== job) return false;
|
||||
job.segment.streaming = textSoFar;
|
||||
return true;
|
||||
}
|
||||
|
||||
fail(segment: TranslationSegment): void {
|
||||
segment.translation = "";
|
||||
segment.state = "failed";
|
||||
complete(job: TranslationJob, translation: string): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.translation = translation;
|
||||
job.segment.translatedSource = job.source;
|
||||
}
|
||||
|
||||
fail(job: TranslationJob): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.failedSource = job.source;
|
||||
}
|
||||
|
||||
private abandonActive(): void {
|
||||
if (this.active) this.active.segment.streaming = null;
|
||||
this.active = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a final to the block before `before` (default: the end) when that
|
||||
* block is final but still waiting for its first request. Returns whether
|
||||
* it did.
|
||||
*/
|
||||
private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean {
|
||||
const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1;
|
||||
const waiting = this.segments[index];
|
||||
if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false;
|
||||
if (this.active?.segment === waiting) return false;
|
||||
waiting.source = `${waiting.source} ${finalText}`;
|
||||
return true;
|
||||
}
|
||||
|
||||
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
|
||||
@@ -87,7 +162,7 @@ export class TranslationModel {
|
||||
let chars = 0;
|
||||
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
|
||||
const earlier = this.segments[index]!;
|
||||
if (earlier.state !== "done" || !earlier.translation) continue;
|
||||
if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue;
|
||||
chars += earlier.source.length + earlier.translation.length;
|
||||
if (turns.length && chars > CONTEXT_CHARS) break;
|
||||
turns.unshift({ source: earlier.source, translation: earlier.translation });
|
||||
@@ -96,9 +171,11 @@ export class TranslationModel {
|
||||
}
|
||||
|
||||
/**
|
||||
* Display lines: each segment's translation, bright; its original, dim,
|
||||
* above it when `showOriginal` is on and in its place until a translation
|
||||
* arrives (or when it failed); then the live partial, dimmer.
|
||||
* Display lines: each block's translation, bright once settled (final, and
|
||||
* of its final text) and dimmer while provisional; its original, dim, above
|
||||
* it when `showOriginal` is on and in its place until a translation
|
||||
* arrives. A redo streams in over the translation it replaces only once it
|
||||
* is as long, so the text doesn't shrink and regrow.
|
||||
*/
|
||||
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
|
||||
const lines: TranslationLine[] = [];
|
||||
@@ -106,21 +183,25 @@ export class TranslationModel {
|
||||
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
|
||||
};
|
||||
for (const segment of this.segments) {
|
||||
const translation = segment.translation.trim();
|
||||
// A finished, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
|
||||
if (translation) push(translation, TRANSLATION_GRAY);
|
||||
if (!segment.source) continue;
|
||||
const settled = segment.final && segment.translatedSource === segment.source;
|
||||
const finished = segment.translation.trim();
|
||||
const streaming = segment.streaming?.trim() ?? "";
|
||||
const translation = streaming.length >= finished.length ? streaming : finished;
|
||||
// A settled, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY);
|
||||
if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY);
|
||||
}
|
||||
if (this.live) push(this.live, LIVE_GRAY);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Plain-text transcript for saving: each original, then its translation. */
|
||||
/** Plain-text transcript for saving: each final original, then its translation. */
|
||||
savedText(): string {
|
||||
return this.segments
|
||||
.filter((segment) => segment.final)
|
||||
.map((segment) => {
|
||||
const translation = segment.translation.trim();
|
||||
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation || translation === segment.source.trim()) return segment.source;
|
||||
return `${segment.source}\n→ ${translation}`;
|
||||
})
|
||||
|
||||
@@ -6,6 +6,7 @@ import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { distanceUnitSetting } from "../../ui/dashboard-settings";
|
||||
import { renderWeatherIcon } from "./weather-icons";
|
||||
import { formatTemperature, widestTemperatureSample } from "./temperature-format";
|
||||
|
||||
@@ -142,7 +143,7 @@ export class WeatherLayer implements Layer {
|
||||
const details: string[] = [];
|
||||
if (current.humidityPercent !== null) details.push(`Humidity ${Math.round(current.humidityPercent)}%`);
|
||||
if (current.windSpeedMph !== null) {
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${Math.round(current.windSpeedMph)} mph`);
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${formatWindSpeed(current.windSpeedMph)}`);
|
||||
}
|
||||
image.drawText(small, textX, cy, details.join(" ") || "Current forecast", 170);
|
||||
cy += lineStep(small);
|
||||
@@ -247,6 +248,11 @@ function forecastRowHeight(font: UiFont): number {
|
||||
return Math.max(FORECAST_MIN_ROW_HEIGHT, tightRowHeight(font) + 3);
|
||||
}
|
||||
|
||||
/** A wind speed (weather data is kept in mph) in the Distance unit setting's speed unit: "12 mph" / "19 km/h". */
|
||||
function formatWindSpeed(mph: number): string {
|
||||
return distanceUnitSetting.get() === "kilometers" ? `${Math.round(mph * 1.609344)} km/h` : `${Math.round(mph)} mph`;
|
||||
}
|
||||
|
||||
function formatAge(ageMs: number): string {
|
||||
const minutes = Math.max(0, Math.round(ageMs / 60_000));
|
||||
if (minutes < 1) return "just now";
|
||||
|
||||
Reference in new issue
Block a user