Port Transcribe to new UI; remove legacy dashboard

This commit is contained in:
James Babcock committed 2026-07-07 22:11:28 -07:00
1 parent 56d0f8c351
commit d0c30f9ca9
21 files changed
+392 -1291

No files matched your search

+95 -404
View File
@@ -14,17 +14,7 @@ import { firmwareIncompatibilityMessage } from "./firmware-compat";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import {
closeDashboardTextSettingEditor,
drawDashboard,
logToDashboard,
receiveDashboardWindowInput,
resetDashboardSleepTimerAndWake,
setDashboardBatteryLevels,
setDashboardActions,
} from "../ui/dashboard";
import { inputEventToString, rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { makeLetterWindowIcon } from "../ui/shell/chrome-layer";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { WorkerAppHost } from "../ui/shell/worker-window";
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app";
import {
@@ -55,6 +45,11 @@ import {
NIGHTSCOUT_WINDOW_ID,
} from "../ui/shell/nightscout-app";
import { createMusicAppWindow, MUSIC_SURFACE_ID, MUSIC_WINDOW_ID } from "../ui/shell/music-app";
import {
createTranscribeAppWindow,
TRANSCRIBE_SURFACE_ID,
TRANSCRIBE_WINDOW_ID,
} from "../ui/shell/transcribe-app";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { APP_VIEWPORT } from "../ui/shell/geometry";
import { type LayerActions } from "../ui/layers";
@@ -72,7 +67,6 @@ import {
voiceProviderSetting,
type ConfigSettingString,
} from "../ui/dashboard-settings";
import { saveRawScreenshot } from "../native/raw-screenshot";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
@@ -94,17 +88,15 @@ export type DashboardSnapshot = {
};
type DashboardListener = (snapshot: DashboardSnapshot) => void;
type LogLevel = "debug"|"info"|"warn"|"error";
const CONTAINER_NAME = "dashboard";
// Compositor surfaces: the dashboard window in the app viewport, and the
// shell chrome (sidebar + top bar + overlays) above it with color-key
// transparency.
const DASHBOARD_WINDOW_SURFACE_ID = "window:dashboard";
// The shell chrome (sidebar + top bar + overlays) composites above all app
// window surfaces with color-key transparency.
const SHELL_SURFACE_ID = "shell";
const DASHBOARD_INTERVAL_MS = 60_000;
// Top-bar clock refresh; the phone-side preview polls the Java composite so
// it reflects every app (including worker apps the TS side never renders).
const SHELL_REFRESH_INTERVAL_MS = 60_000;
const PREVIEW_INTERVAL_MS = 1_000;
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
const TRANSCRIBE_RENDER_INTERVAL_MS = 250;
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
@@ -119,32 +111,6 @@ function formatTimestamp(date: Date): string {
return date.toISOString().slice(11, 23);
}
function pad2(value: number): string {
return String(value).padStart(2, "0");
}
function formatTime24h(date: Date, includeSeconds: boolean): string {
const hours = date.getHours();
const minutes = pad2(date.getMinutes());
const seconds = pad2(date.getSeconds());
return includeSeconds
? `${hours}:${minutes}:${seconds}`
: `${hours}:${minutes}`;
}
function formatUtcOffset(date: Date): string {
const offsetMinutes = -date.getTimezoneOffset();
const sign = offsetMinutes >= 0 ? "+" : "-";
const absoluteMinutes = Math.abs(offsetMinutes);
const hours = Math.floor(absoluteMinutes / 60);
const minutes = absoluteMinutes % 60;
return `UTC${sign}${pad2(hours)}:${pad2(minutes)}`;
}
function formatForStatus(date: Date): string {
return formatTime24h(date, true);
}
function eventName(eventType: number): string {
return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`;
}
@@ -153,25 +119,6 @@ function sourceName(eventSource: number): string {
return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`;
}
function normalizeContainerName(name: string): string {
return name.replace(/[^\x20-\x7e]+/g, "");
}
function isDashboardContainerName(name: string): boolean {
const normalized = normalizeContainerName(name);
return normalized === CONTAINER_NAME || normalized.includes(CONTAINER_NAME);
}
/** Top-left viewport-sized crop of a full-lens dashboard paint. */
function cropToViewport(image: GrayImage): GrayImage {
if (image.width === APP_VIEWPORT.width && image.height === APP_VIEWPORT.height) {
return image;
}
const cropped = new GrayImage(APP_VIEWPORT.width, APP_VIEWPORT.height, 0);
cropped.bitBlt(image, 0, 0, { width: APP_VIEWPORT.width, height: APP_VIEWPORT.height });
return cropped;
}
class DashboardController {
private phase: ConnectionPhase = "disconnected";
private status = "Disconnected.";
@@ -185,9 +132,9 @@ class DashboardController {
private readonly listeners = new Set<DashboardListener>();
private communicator: FaceclawCommunicatorBridge | null = null;
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
private shellRefreshTimer: ReturnType<typeof setInterval> | null = null;
private previewTimer: ReturnType<typeof setInterval> | null = null;
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
private transcribeRenderTimer: ReturnType<typeof setInterval> | null = null;
private offState: (() => void) | null = null;
private offLog: (() => void) | null = null;
private offRing: (() => void) | null = null;
@@ -195,23 +142,14 @@ class DashboardController {
private offEvenAppConflict: (() => void) | null = null;
private offFrameMetrics: (() => void) | null = null;
private offFirmwareInfo: (() => void) | null = null;
private offMedia: (() => void) | null = null;
private offNightscout: (() => void) | null = null;
private offVoiceStatus: (() => void) | null = null;
private offVoiceWakeWord: (() => void) | null = null;
private offAndroidNotification: (() => void) | null = null;
private lastInput = "waiting...";
private lastSys = "none yet";
private renderInProgress = false;
private renderQueued = false;
private queuedRenderReason: "initial" | "interval" = "interval";
private queuedFrameId = 0;
private shellRenderInProgress = false;
private shellRenderQueued = false;
private nextShellRenderWantsFreshData = false;
// Last painted surface images, for compositing the phone-side preview.
private lastShellImage: GrayImage | null = null;
private lastWindowImage: GrayImage | null = null;
// One shared worker per app hosts all its windows; spawned on first launch.
private readonly appHosts = new Map<string, WorkerAppHost>();
private nextWindowSerial = 1;
@@ -223,29 +161,19 @@ class DashboardController {
private sharedActions!: Omit<LayerActions, "requestRender">;
private lastForegroundNotificationUpdateAtMs = 0;
private lastConnectedPreviewUpdateAtMs = 0;
// Consumed by the next renderDashboard: repaint with data sources not
// allowed to serve stale caches (set after a frame painted with stale data).
private nextRenderWantsFreshData = false;
constructor() {
const sharedActions = {
disconnect: () => this.disconnect(),
startTextSettingEdit: (setting: ConfigSettingString) => this.startTextSettingEdit(setting),
endTextSettingEdit: () => this.endTextSettingEdit(),
setTranscribeRenderActive: (active: boolean) => this.setTranscribeRenderActive(active),
startVoiceCapture: () => this.startVoiceCapture(),
stopVoiceCapture: () => this.stopVoiceCapture(),
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
setDashboardActions({
...sharedActions,
requestRender: () => {
void this.requestRender("interval").catch((error) => {
this.appendLog(`requested render failed: ${this.formatError(error)}`);
});
},
});
shell.configure({
actions: {
...sharedActions,
@@ -282,6 +210,7 @@ class DashboardController {
{ appId: "teleprompt", label: "Teleprompt" },
{ appId: "music", label: "Music" },
{ appId: "nightscout", label: "Nightscout" },
{ appId: "transcribe", label: "Transcribe" },
{ appId: "notifications", label: "Notifications" },
{ appId: "debug-tests", label: "Debug tests" },
{ appId: "settings", label: "Settings" },
@@ -294,26 +223,6 @@ class DashboardController {
},
}),
);
shell.registerWindow({
appId: "dashboard",
windowId: "dashboard",
title: "Dashboard",
surfaceId: DASHBOARD_WINDOW_SURFACE_ID,
closeable: false,
drawIcon: makeLetterWindowIcon("D"),
handleInput: async (event, frameId) => {
await receiveDashboardWindowInput(event);
await this.requestRender("interval", frameId);
},
requestRender: () => {
void this.requestRender("interval").catch((error) => {
this.appendLog(`window render failed: ${this.formatError(error)}`);
});
},
setForeground: (foreground) => {
this.setWindowSurfaceVisible(DASHBOARD_WINDOW_SURFACE_ID, foreground);
},
});
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => {
void this.handleAndroidNotificationPosted(notificationKey).catch((error) => {
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
@@ -336,12 +245,12 @@ class DashboardController {
}
/**
* Save the current dashboard as the raw headerless 4bpp frame buffer the
* wire compressor sees; test data for compression experiments.
* Save the current composited screen as the raw headerless 4bpp frame
* buffer the wire compressor sees; test data for compression experiments.
*/
saveRawDashboardScreenshot(): string {
const path = saveRawScreenshot(drawDashboard());
this.appendLog(`raw screenshot saved: ${path}`);
const path = this.communicator?.saveRawCompositeScreenshot() ?? "";
this.appendLog(path ? `raw screenshot saved: ${path}` : "raw screenshot skipped: not connected");
return path;
}
@@ -433,7 +342,7 @@ class DashboardController {
// truth while typing; echoing activeTextSettingValue back into its two-way
// binding on every keystroke drops fast/pasted characters (observed: a
// 51-char API key stored as its first 46 chars). Just refresh the preview.
this.previewOrRenderAfterTextSettingChange(setting.label);
this.previewOrRenderAfterTextSettingChange();
}
async connect(): Promise<void> {
@@ -452,7 +361,6 @@ class DashboardController {
this.lastSys = "none yet";
this.firmwareWarningMessage = "";
this.refreshBatteryOptimizationStatus();
resetDashboardSleepTimerAndWake();
this.refreshEvenAppStatus();
this.setPhase("connecting");
this.setStatus("Connecting to the glasses...");
@@ -505,20 +413,13 @@ class DashboardController {
});
});
this.offBattery = communicator.onBatteryState((state) => {
setDashboardBatteryLevels({
headset: state.battery,
headsetCharging: state.chargingStatus > 0,
});
shell.setBatteryLevels({
headset: state.battery,
headsetCharging: state.chargingStatus > 0,
});
if ((this.phase === "connected" || this.phase === "charging") && this.communicator) {
// Repaint the top bar (battery indicators live in the shell chrome).
this.requestShellRender();
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
this.appendLog(`battery update failed: ${message}`);
});
}
});
this.offEvenAppConflict = communicator.onEvenAppConflict((message) => {
@@ -550,22 +451,8 @@ class DashboardController {
this.emit();
}
});
this.offMedia = mediaControllerBridge.onStateChange(() => {
if (this.phase === "connected" && this.communicator) {
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
this.appendLog(`media update failed: ${message}`);
});
}
});
this.offNightscout = nightscoutBridge.onStateChange(() => {
if (this.phase === "connected" && this.communicator) {
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
this.appendLog(`nightscout update failed: ${message}`);
});
}
});
// The Music and Nightscout apps subscribe to their bridges directly and
// repaint their own windows, so bridge updates need no controller action.
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
this.appendLog(state.status);
});
@@ -577,8 +464,8 @@ class DashboardController {
await mediaControllerBridge.start();
await nightscoutBridge.start();
// Register the compositor surfaces: the dashboard window in the app
// viewport, and the shell chrome above it.
// Register the compositor surfaces: the shell chrome above all windows,
// and a surface per live window (only the foreground one is composited).
await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
await communicator.configureSurface(SHELL_SURFACE_ID, {
x: 0,
@@ -588,8 +475,6 @@ class DashboardController {
zOrder: 1,
transparency: "color-key",
});
// Window surfaces for every live window (including any launched while
// disconnected); only the foreground window's surface is composited.
const foregroundWindowId = shell.foregroundWindow()?.windowId;
for (const window of shell.getWindows()) {
await this.configureWindowSurface(window.surfaceId, window.windowId === foregroundWindowId);
@@ -597,25 +482,19 @@ class DashboardController {
await communicator.start();
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
await this.requestRender("initial");
this.dashboardTimer = setInterval(() => {
// Also refreshes the top-bar clock on the shell surface.
// Refresh the top-bar clock and the phone-side preview once a minute,
// and keep the Android persistent notification current.
this.shellRefreshTimer = setInterval(() => {
this.requestShellRender();
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
this.setStatus(`Dashboard update failed: ${message}`);
this.appendLog(`dashboard update failed: ${message}`);
});
}, DASHBOARD_INTERVAL_MS);
this.updateCompositePreview();
this.updateConnectedForegroundNotification();
}, SHELL_REFRESH_INTERVAL_MS);
this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS);
this.screenTimeoutTimer = setInterval(() => {
if (this.phase !== "connected" || !this.communicator) return;
if (!shell.applyScreenTimeout()) return;
this.endTextSettingEdit();
this.requestShellRender();
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
this.appendLog(`screen timeout render failed: ${message}`);
});
}, SCREEN_TIMEOUT_CHECK_MS);
} catch (error) {
const message = this.formatError(error);
@@ -633,18 +512,10 @@ class DashboardController {
this.offFrameMetrics = null;
this.offFirmwareInfo?.();
this.offFirmwareInfo = null;
this.offMedia?.();
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
@@ -681,10 +552,6 @@ class DashboardController {
this.offFrameMetrics = null;
this.offFirmwareInfo?.();
this.offFirmwareInfo = null;
this.offMedia?.();
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
@@ -706,10 +573,6 @@ class DashboardController {
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
await communicator?.close().catch(() => {});
} finally {
stopForegroundNotification();
@@ -754,6 +617,22 @@ class DashboardController {
* gate even though the audio source is the G2 mic over BLE.
*/
private startVoiceCapture(): void {
this.beginVoiceCapture("ptt");
}
private stopVoiceCapture(): void {
voiceControlBridge.stopPushToTalk();
}
private startContinuousVoiceCapture(): void {
this.beginVoiceCapture("continuous");
}
private stopContinuousVoiceCapture(): void {
voiceControlBridge.stopContinuousCapture();
}
private beginVoiceCapture(kind: "ptt" | "continuous"): void {
if (this.phase !== "connected" || !this.communicator) {
return;
}
@@ -761,22 +640,23 @@ class DashboardController {
void ensureVoicePermissions()
.then(() => {
if (this.phase !== "connected" || this.communicator !== communicator) return;
voiceControlBridge.startPushToTalk({
const options = {
communicator: communicator.getNativeCommunicator(),
provider: voiceProviderSetting.get(),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
});
};
if (kind === "ptt") {
voiceControlBridge.startPushToTalk(options);
} else {
voiceControlBridge.startContinuousCapture(options);
}
})
.catch((error) => {
this.appendLog(`voice permission failed: ${this.formatError(error)}`);
});
}
private stopVoiceCapture(): void {
voiceControlBridge.stopPushToTalk();
}
private endTextSettingEdit(): void {
const finishedSetting = this.activeTextSetting;
this.activeTextSetting = null;
@@ -788,23 +668,14 @@ class DashboardController {
/**
* Finish the active edit from the phone side (e.g. the IME's done key):
* ends the edit session and navigates the glasses out of the edit page,
* in whichever window hosts it (the Settings app or the dashboard).
* ends the edit session and navigates the Settings app's glasses editor
* out of the edit page.
*/
finishActiveTextSettingEdit(): void {
if (!this.activeTextSetting) return;
this.endTextSettingEdit();
if (this.settingsApp?.closeTextEditor()) {
this.settingsApp.requestRender();
return;
}
if (!closeDashboardTextSettingEditor()) return;
if (this.phase === "connected" && this.communicator) {
void this.requestRender("interval").catch((error) => {
this.appendLog(`edit close render failed: ${this.formatError(error)}`);
});
} else {
this.updateDisplayPreviewFromImage(drawDashboard());
}
}
@@ -812,7 +683,7 @@ class DashboardController {
if (setting.get() !== value) {
setting.set(value);
this.emit();
this.previewOrRenderAfterTextSettingChange(setting.label);
this.previewOrRenderAfterTextSettingChange();
}
}
@@ -820,151 +691,13 @@ class DashboardController {
await nightscoutBridge.refreshNow().catch((error) => {
this.appendLog(`nightscout settings refresh failed: ${this.formatError(error)}`);
});
if (this.phase === "connected" && this.communicator) {
await this.requestRender("interval").catch((error) => {
this.appendLog(`nightscout settings render failed: ${this.formatError(error)}`);
});
} else {
const image = drawDashboard();
this.updateDisplayPreviewFromImage(image);
}
}
private previewOrRenderAfterTextSettingChange(label: string): void {
// Echo phone-side keystrokes into the glasses editor, wherever it lives.
private previewOrRenderAfterTextSettingChange(): void {
// Echo phone-side keystrokes into the Settings app's glasses editor.
if (this.settingsApp?.isTextEditorOnTop()) {
this.settingsApp.requestRender();
return;
}
if (this.phase === "connected" && this.communicator) {
void this.requestRender("interval").catch((error) => {
this.appendLog(`${label} update failed: ${this.formatError(error)}`);
});
return;
}
const image = drawDashboard();
this.updateDisplayPreviewFromImage(image);
}
/**
* Serialize renders; frameId identifies the input event or timer tick that
* asked for this render (one is started here for callers that don't have
* one). Ownership of the frame passes to renderDashboard and then to the
* Java side; requests that coalesce into an already-queued render finish
* immediately as discarded. When coalescing we keep the oldest waiting frame
* so measured latency reflects the worst-served request.
*/
private async requestRender(reason: "initial" | "interval", frameId?: number): Promise<void> {
const newFrameId = frameId ?? frameTimings.startFrame(`render:${reason}`);
this.queuedRenderReason = this.queuedRenderReason === "initial" ? "initial" : reason;
if (this.renderInProgress) {
if (this.renderQueued && this.queuedFrameId > 0) {
frameTimings.finishFrame(newFrameId, `discarded: coalesced into frame#${this.queuedFrameId}`);
} else {
this.renderQueued = true;
this.queuedFrameId = newFrameId;
frameTimings.logFrame(newFrameId, "render queued behind in-progress render");
}
return;
}
this.renderInProgress = true;
let frameToRender = newFrameId;
try {
while (true) {
const nextReason = this.queuedRenderReason;
this.renderQueued = false;
this.queuedRenderReason = "interval";
try {
await this.renderDashboard(nextReason, frameToRender);
} catch (error) {
frameTimings.finishFrame(frameToRender, `discarded: render failed: ${this.formatError(error)}`);
throw error;
}
if (!this.renderQueued || this.queuedFrameId <= 0) break;
frameToRender = this.queuedFrameId;
this.queuedFrameId = 0;
}
} finally {
this.renderInProgress = false;
}
}
private async renderDashboard(reason: "initial" | "interval", frameId: number): Promise<void> {
console.log("renderDashboard", reason);
frameTimings.logFrame(frameId, `renderDashboard start (${reason})`);
const wantFreshData = this.nextRenderWantsFreshData;
this.nextRenderWantsFreshData = false;
beginRenderPass(!wantFreshData);
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => drawDashboard()),
);
const paintMs = Date.now() - paintStartedAtMs;
const paintUsedStaleData = endRenderPass();
if (paintUsedStaleData) {
frameTimings.logFrame(frameId, "painted with stale data; will schedule a fresh-data repaint");
}
// The dashboard still paints the full lens; its window surface only shows
// the viewport-sized top-left crop (content near the right/bottom edges
// clips until the legacy layouts are resized).
const cropped = frameTimings.span(frameId, "crop", () => cropToViewport(image));
const fingerprint = frameTimings.span(frameId, "fingerprint", () => cropped.fingerprint());
this.lastWindowImage = cropped;
const updatePreviewAfterTransmit = this.phase === "connected";
if (!updatePreviewAfterTransmit) {
this.updateDisplayPreviewFromImage(image);
}
if (this.communicator && this.phase === "charging") {
// Glasses are in the charging case; keep the phone preview fresh but
// send nothing.
frameTimings.finishFrame(frameId, "discarded: glasses charging");
this.updateConnectedForegroundNotification();
console.log("renderDashbaord finished (charging)");
return;
}
if (this.communicator) {
if (shell.isScreenOn()) {
await this.communicator.setG2ScreenOn(true);
}
console.log("submitSurfaceFrame");
const buffer = frameTimings.span(frameId, "to8bpp", () => cropped.to8bppBuffer());
// The Java side owns the frame from here: it finishes it on last-packet
// ack, dedup, supersede, or timeout.
await this.communicator.submitSurfaceFrame(
DASHBOARD_WINDOW_SURFACE_ID,
buffer,
{ x: 0, y: 0, width: cropped.width, height: cropped.height },
fingerprint,
paintMs,
frameId,
);
const outcome = await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
if (outcome === null) {
frameTimings.logFrame(frameId, "transmit backpressure wait timed out; render loop continuing");
}
if (!shell.isScreenOn()) {
await this.communicator.setG2ScreenOn(false);
}
if (updatePreviewAfterTransmit) {
this.updateConnectedCompositePreview();
}
} else {
frameTimings.finishFrame(frameId, "discarded: no communicator, preview only");
}
if (paintUsedStaleData) {
this.nextRenderWantsFreshData = true;
void this.requestRender("interval").catch((error) => {
this.appendLog(`fresh-data repaint failed: ${this.formatError(error)}`);
});
}
this.updateConnectedForegroundNotification();
if (reason === "initial") {
this.appendLog("initial dashboard image queued");
}
console.log("renderDashboard finished");
}
private async handleInputEvent(event: RawInputEvent): Promise<void> {
@@ -974,7 +707,6 @@ class DashboardController {
let frameOwned = false;
try {
const inputEvent = rawInputEventToInputEvent(event);
logToDashboard(inputEventToString(inputEvent));
frameTimings.spanStart(frameId, "handle-input");
let outcome: ShellInputOutcome;
try {
@@ -988,7 +720,6 @@ class DashboardController {
if (event.kind === "sys-event") {
this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`;
this.appendLog(`sys-event ${this.lastSys}`);
if (
event.eventType === OsEventTypeList.FOREGROUND_EXIT_EVENT ||
event.eventType === OsEventTypeList.ABNORMAL_EXIT_EVENT ||
@@ -996,19 +727,9 @@ class DashboardController {
) {
this.appendLog("display state invalidated by firmware exit event");
}
if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
this.lastInput = eventName(event.eventType);
}
} else if (event.kind === "text-click" && isDashboardContainerName(event.containerName)) {
if (
event.eventType === OsEventTypeList.SCROLL_TOP_EVENT ||
event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT
) {
this.lastSys = `TEXT/${eventName(event.eventType)}`;
this.lastInput = `TEXT_${eventName(event.eventType)}`;
this.appendLog(`text-event ${this.lastSys}`);
}
}
if (outcome.shell) {
@@ -1144,6 +865,16 @@ class DashboardController {
await this.launchInProcessApp(MUSIC_WINDOW_ID, MUSIC_SURFACE_ID, createMusicAppWindow);
return;
}
if (appId === "transcribe") {
await this.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) =>
createTranscribeAppWindow({
...options,
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
}),
);
return;
}
if (appId === "teleprompt") {
await this.launchInProcessApp(TELEPROMPT_WINDOW_ID, TELEPROMPT_SURFACE_ID, (options) =>
createTelepromptBrowserWindow({
@@ -1268,7 +999,6 @@ class DashboardController {
this.nextShellRenderWantsFreshData = true;
this.requestShellRender();
}
this.lastShellImage = image;
if (!this.communicator || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: shell render with no active connection");
return;
@@ -1284,24 +1014,15 @@ class DashboardController {
frameId,
);
await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
this.updateConnectedCompositePreview();
this.updateCompositePreview();
}
private async handleWakeWord(keyword: string): Promise<void> {
const normalized = keyword.trim();
if (normalized.length === 0) return;
this.appendLog(`wake-word detected: ${normalized}`);
const changed = resetDashboardSleepTimerAndWake();
if (this.phase === "connected" && this.communicator) {
if (changed) {
this.requestShellRender();
await this.requestRender("initial");
}
return;
}
if (changed) {
const image = drawDashboard();
this.updateDisplayPreviewFromImage(image);
if (shell.wake("sidebar")) {
this.requestShellRender();
}
}
@@ -1321,9 +1042,6 @@ class DashboardController {
}
shell.openNotificationModal(notificationKey, wokeScreen);
this.requestShellRender();
if (this.phase === "connected" && this.communicator) {
await this.requestRender("interval");
}
}
private async playBuzzerSequence(payload: Uint8Array): Promise<void> {
@@ -1333,35 +1051,19 @@ class DashboardController {
await this.communicator.playBuzzerSequence(payload);
}
private setTranscribeRenderActive(active: boolean): void {
if (!active || this.phase !== "connected" || !this.communicator) {
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
return;
}
if (this.transcribeRenderTimer) return;
this.transcribeRenderTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => {
this.appendLog(`transcribe render failed: ${this.formatError(error)}`);
});
}, TRANSCRIBE_RENDER_INTERVAL_MS);
}
private clearDashboardTimer(): void {
if (this.dashboardTimer) {
clearInterval(this.dashboardTimer);
this.dashboardTimer = null;
if (this.shellRefreshTimer) {
clearInterval(this.shellRefreshTimer);
this.shellRefreshTimer = null;
}
if (this.previewTimer) {
clearInterval(this.previewTimer);
this.previewTimer = null;
}
if (this.screenTimeoutTimer) {
clearInterval(this.screenTimeoutTimer);
this.screenTimeoutTimer = null;
}
if (this.transcribeRenderTimer) {
clearInterval(this.transcribeRenderTimer);
this.transcribeRenderTimer = null;
}
}
private setPhase(phase: ConnectionPhase): void {
@@ -1402,27 +1104,13 @@ class DashboardController {
this.emit();
}
private updateDisplayPreviewFromImage(image: GrayImage): void {
this.setDisplayPreview(grayImageToPreviewSource(image));
}
/**
* Phone-side preview of what is actually on the glasses: the window surface
* in its viewport with the shell chrome composited over it, mirroring the
* Java compositor.
* Phone-side preview of what is on the glasses, fetched from the Java
* compositor so it reflects every surface (chrome + whichever app is
* foreground, including worker apps the TS side never renders). Throttled
* to avoid rebuilding the bitmap faster than the phone UI needs it.
*/
private buildCompositePreviewImage(): GrayImage {
const preview = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
if (this.lastWindowImage) {
preview.bitBlt(this.lastWindowImage, APP_VIEWPORT.x, APP_VIEWPORT.y);
}
if (this.lastShellImage) {
preview.bitBlt(this.lastShellImage, 0, 0, { transparentZero: true });
}
return preview;
}
private updateConnectedCompositePreview(): void {
private updateCompositePreview(): void {
if (!this.communicator) return;
const now = Date.now();
if (
@@ -1432,7 +1120,10 @@ class DashboardController {
return;
}
this.lastConnectedPreviewUpdateAtMs = now;
this.updateDisplayPreviewFromImage(this.buildCompositePreviewImage());
const preview = this.communicator.getCompositePreview();
if (preview) {
this.setDisplayPreview(preview);
}
}
private formatError(error: unknown): string {
@@ -1448,7 +1139,7 @@ class DashboardController {
case "click":
return {
kind: "sys-event",
containerName: CONTAINER_NAME,
containerName: "",
eventType: OsEventTypeList.CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
systemExitReasonCode: 0,
@@ -1457,7 +1148,7 @@ class DashboardController {
case "double-click":
return {
kind: "sys-event",
containerName: CONTAINER_NAME,
containerName: "",
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
systemExitReasonCode: 0,
@@ -1466,7 +1157,7 @@ class DashboardController {
case "scroll-up":
return {
kind: "text-click",
containerName: CONTAINER_NAME,
containerName: "",
eventType: OsEventTypeList.SCROLL_TOP_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
systemExitReasonCode: 0,
@@ -1476,7 +1167,7 @@ class DashboardController {
default:
return {
kind: "text-click",
containerName: CONTAINER_NAME,
containerName: "",
eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
systemExitReasonCode: 0,
+16 -1
View File
@@ -1,8 +1,10 @@
import { Utils } from "@nativescript/core";
import { ImageSource, Utils } from "@nativescript/core";
import * as frameTimings from "./frame-timings";
declare const com: any;
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
export type CommunicatorPhase =
| "disconnected"
| "connecting"
@@ -338,6 +340,19 @@ export class FaceclawCommunicatorBridge {
});
}
/** Phone-UI preview of the current composited screen, or null if none yet. */
getCompositePreview(): ImageSource | null {
if (!global.isAndroid) return null;
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA);
return bitmap ? new ImageSource(bitmap) : null;
}
/** Save the current composite as a raw 4bpp screenshot; returns the path (empty if none). */
saveRawCompositeScreenshot(): string {
if (!global.isAndroid) return "";
return String(this.communicator.saveRawCompositeScreenshot());
}
/**
* Create or reconfigure a compositor surface. Geometry changes take effect
* when the next frame is submitted.
+24
View File
@@ -73,6 +73,30 @@ export function listDirectory(path: string): DirectoryEntry[] | null {
}
}
/** Write UTF-8 text to a file in the public Downloads directory; returns its path or null. */
export function writeTextToDownloads(filename: string, text: string): string | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
const file = new java.io.File(downloads, filename);
const bytes = new java.lang.String(text).getBytes("UTF-8");
// FileOutputStream rather than Files.write to avoid NativeScript varargs /
// overload resolution pitfalls.
const stream = new java.io.FileOutputStream(file);
try {
stream.write(bytes);
} finally {
stream.close();
}
return String(file.getAbsolutePath());
} catch (error) {
console.warn(`writeTextToDownloads failed for ${filename}: ${error}`);
return null;
}
}
/** Read a UTF-8 text file (size-capped); null when unreadable or too large. */
export function readTextFile(path: string): string | null {
try {
+52 -7
View File
@@ -27,6 +27,9 @@ export type PushToTalkOptions = {
saveRecording: boolean;
};
/** Who currently wants the mic running. */
type CaptureHolder = "ptt" | "continuous";
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
@@ -35,6 +38,11 @@ export class FaceclawVoiceControlBridge {
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
private started = false;
// The mic is a single shared stream; these are the reasons it is running.
// The first holder starts capture (choosing the provider); the mic stops
// when the last one releases. Transcript events broadcast to every
// listener, so push-to-talk and the Transcribe window both receive text.
private readonly captureHolders = new Set<CaptureHolder>();
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
private cloudClient: ElevenLabsSttClient | null = null;
@@ -54,10 +62,34 @@ export class FaceclawVoiceControlBridge {
return () => this.transcriptListeners.delete(listener);
}
/** Begin push-to-talk capture with the selected provider. */
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
}
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
stopPushToTalk(): void {
this.releaseCapture("ptt", true);
}
/** Begin continuous capture (Transcribe): the mic stays on until released. */
startContinuousCapture(options: PushToTalkOptions): void {
this.acquireCapture("continuous", options);
}
stopContinuousCapture(): void {
this.releaseCapture("continuous", false);
}
private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void {
if (!global.isAndroid) return;
if (this.started) this.stop();
const wasIdle = this.captureHolders.size === 0;
this.captureHolders.add(holder);
if (!wasIdle) {
// Mic already running; the new holder just shares the existing stream
// (transcripts are already broadcast to its listeners).
return;
}
this.ensureController();
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(options.saveRecording);
@@ -83,21 +115,34 @@ export class FaceclawVoiceControlBridge {
this.controller?.start("onboard");
}
/** End push-to-talk: stop the mic and, for cloud, commit for a final result. */
stopPushToTalk(): void {
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
if (!this.captureHolders.delete(holder)) {
// Never held; still finish a dangling cloud commit if one is pending.
if (commit) this.cloudClient?.finish();
return;
}
if (this.captureHolders.size > 0) {
// Another holder still wants the mic; keep it running.
return;
}
if (!this.started) {
this.cloudClient?.finish();
return;
}
// Order matters for cloud: stopping the Java controller flushes any final
// decode/PCM; then commit so ElevenLabs finalizes the transcript. The
// socket is closed once the committed transcript arrives, or on stop().
// decode/PCM; then commit so ElevenLabs finalizes the transcript.
this.controller?.stop();
this.started = false;
this.cloudClient?.finish();
if (commit) {
this.cloudClient?.finish();
} else {
this.cloudClient?.stop();
this.cloudClient = null;
}
}
stop(): void {
this.captureHolders.clear();
if (global.isAndroid) {
this.controller?.stop();
}
-71
View File
@@ -1,71 +0,0 @@
import takeRight from "lodash/takeRight";
import { getDefaultSmallFont } from "~/graphics/bdffont";
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
import { Layer, LayerContext, type DashboardInputEvent } from "../layers";
import { GrayImage } from "~/graphics/image";
export class DebugLogDashboardPlugin extends DashboardPlugin {
constructor() {
super({
id: "input-debug-log",
label: "Input debug log",
});
}
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
const font = getDefaultSmallFont();
image.drawText(font, bounds.x + 10, bounds.y + 14, "Input log", 180);
const lineHeight = 14;
const visibleLineCount = Math.max(1, Math.floor((bounds.height - 28) / lineHeight));
const visibleLines = takeRight(state.logLines, visibleLineCount);
for (let i = 0; i < visibleLines.length; i++) {
image.drawText(font, bounds.x + 8, bounds.y + 26 + i * lineHeight, visibleLines[i]!, 190);
}
}
override createFullscreenLayer(getState: () => DashboardPluginState): Layer {
return new DebugLogLayer(getState);
}
}
class DebugLogLayer implements Layer {
private scrollOffset = 0;
constructor(private readonly getState: () => DashboardPluginState) {}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const image = new GrayImage(576, 288, 0);
image.drawRect(12, 12, 552, 264, 52);
image.drawText(font, 22, 16, "Input debug log", 220);
const logs = this.getState().logLines;
const lineHeight = 14;
const visibleCount = Math.max(1, Math.floor((288 - 64) / lineHeight));
const maxOffset = Math.max(0, logs.length - visibleCount);
this.scrollOffset = Math.max(0, Math.min(this.scrollOffset, maxOffset));
const start = Math.max(0, maxOffset - this.scrollOffset);
const visible = logs.slice(start, start + visibleCount);
for (let index = 0; index < visible.length; index++) {
image.drawText(font, 22, 42 + index * lineHeight, visible[index]!, 180);
}
image.drawText(font, 22, 252, "Scroll: browse Double-click: back", 110);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
this.scrollOffset += 1;
return;
case "scroll-down":
this.scrollOffset = Math.max(0, this.scrollOffset - 1);
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
}
-59
View File
@@ -1,59 +0,0 @@
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { GrayImage } from "../../graphics/image";
import { wrapText } from "../../graphics/textwrap";
import { mediaControllerBridge, type MediaControllerState } from "../../native/media-controller";
export class MusicControllerDashboardPlugin extends DashboardPlugin {
constructor() {
super({
id: "music-controller",
label: "Music controller",
});
}
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
const font = getDefaultSmallFont();
const media = state.media;
image.drawText(font, bounds.x + 10, bounds.y + 14, "Now playing", 180);
if (!media.accessEnabled) {
image.drawText(font, bounds.x + 10, bounds.y + 30, "Enable notification", 150);
image.drawText(font, bounds.x + 10, bounds.y + 44, "access for media", 150);
return;
}
if (!media.available) {
image.drawText(font, bounds.x + 10, bounds.y + 30, "No active session", 150);
return;
}
const titleLines = wrapText(font, media.title || "Unknown title", bounds.width - 20).slice(0, 2);
const artistLine = media.artist || media.album || media.packageName || "Unknown source";
image.drawText(font, bounds.x + 10, bounds.y + 30, titleLines[0] ?? "", 210);
if (titleLines[1]) {
image.drawText(font, bounds.x + 10, bounds.y + 44, titleLines[1], 210);
}
image.drawText(font, bounds.x + 10, bounds.y + 62, artistLine, 160);
image.drawText(font, bounds.x + 10, bounds.y + bounds.height - 18, playbackLabel(media), 130);
}
}
function playbackLabel(media: MediaControllerState): string {
switch (media.playbackState) {
case "playing":
return "Playing";
case "paused":
return "Paused";
case "buffering":
return "Buffering";
case "stopped":
return "Stopped";
case "notification-access-required":
return "Access required";
default:
return media.status || "Idle";
}
}
+3 -83
View File
@@ -1,8 +1,8 @@
import { BdfFont, getDefaultLargeFont, getDefaultSmallFont } from "../../graphics/bdffont";
import { GrayImage, imageFromAsciiArt } from "../../graphics/image";
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
import { DashboardInputEvent, Layer, LayerContext } from "../layers";
import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge";
import { isNightscoutSettingsConfigured } from "../dashboard-settings";
import { formatAgeShortFromTimestamp, formatTimestamp } from "~/util/date-util";
const nightscoutLargeFont = getDefaultLargeFont();
@@ -10,84 +10,6 @@ const NIGHTSCOUT_STALE_MS = 15 * 60 * 1000;
const NIGHTSCOUT_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000;
const NIGHTSCOUT_GRAPH_TIME_QUANTUM_MS = 60 * 1000;
export class NightscoutDashboardPlugin extends DashboardPlugin {
constructor() {
super({
id: "nightscout",
label: "Nightscout",
});
}
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
const font = getDefaultSmallFont();
const nightscout = state.nightscout;
const nowMs = Date.now();
image.drawText(font, bounds.x + 10, bounds.y + 10, "Nightscout", 180);
if (!state.nightscoutConfigured || nightscout.configurationMissing) {
image.drawText(font, bounds.x + 10, bounds.y + 30, "Needs setup", 170);
image.drawText(font, bounds.x + 10, bounds.y + 44, "Click to configure", 140);
return;
}
if (!nightscout.available || !nightscout.latest) {
image.drawText(font, bounds.x + 10, bounds.y + 30, "No glucose data", 150);
image.drawText(font, bounds.x + 10, bounds.y + 44, truncateLine(nightscout.status, 26), 120);
return;
}
const latest = nightscout.latest;
const deltaLabel = formatDelta(nightscout.delta);
const glucoseText = `${latest.sgv}`;
const glucoseX = bounds.x + bounds.width - 92;
image.drawText(nightscoutLargeFont, glucoseX, bounds.y + 10, glucoseText, 230);
if (isNightscoutPointStale(latest, nowMs)) {
drawNightscoutValueStrikeThrough(image, glucoseX, bounds.y + 22, nightscoutLargeFont.measureText(glucoseText));
}
drawNightscoutDelta(image, font, bounds.x + bounds.width - 92, bounds.y + 30, deltaLabel, nightscout.direction, 160);
image.drawText(
font,
bounds.x + 10,
bounds.y + 30,
`IOB ${nightscout.iob === null ? "--" : nightscout.iob.toFixed(1)} COB ${nightscout.cob === null ? "--" : formatWholeNumber(nightscout.cob)}`,
160,
);
image.drawText(
font,
bounds.x + 10,
bounds.y + 44,
`Loop ${nightscout.openapsStatusShort} CAGE ${formatAgeShortFromTimestamp(nightscout.cageTimestampMs, nowMs)}`,
138,
);
drawNightscoutGraph(
image,
{
x: bounds.x + 10,
y: bounds.y + 60,
width: bounds.width - 20,
height: Math.max(16, bounds.height - 82),
},
nightscout,
nowMs,
font,
);
}
}
function drawNightscoutDelta(
image: GrayImage,
font: BdfFont,
x: number,
y: number,
deltaLabel: string,
direction: string,
shade: number,
): void {
const text = direction ? `${deltaLabel} ` : deltaLabel;
image.drawText(font, x, y, text, shade);
drawDirectionIndicator(image, font, x + font.measureText(text), y, direction, shade);
}
function drawDirectionIndicator(
image: GrayImage,
font: BdfFont,
@@ -364,20 +286,18 @@ function drawNightscoutReferenceLine(
export class NightscoutLayer implements Layer {
constructor(private readonly getState: () => DashboardPluginState) {}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
// Sized to the hosting stack (the Nightscout app viewport).
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const nightscout = this.getState().nightscout;
const nightscout = nightscoutBridge.snapshot();
const nowMs = Date.now();
const footerY = height - 26;
image.drawRect(12, 12, width - 24, height - 24, 52);
image.drawText(font, 22, 16, "Nightscout", 220);
if (!this.getState().nightscoutConfigured || nightscout.configurationMissing) {
if (!isNightscoutSettingsConfigured() || nightscout.configurationMissing) {
image.drawText(font, 22, 44, "Nightscout needs configuration.", 180);
image.drawText(font, 22, 62, "Use Settings > Integrations > Nightscout.", 140);
return image;
+52 -21
View File
@@ -1,50 +1,65 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
/**
* Live transcription view. Continuous mic capture is owned by the app wrapper
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
* it, and saves it to Downloads on click.
*/
export class TranscribeLayer implements Layer {
private status = "Starting transcription...";
private status = "Listening...";
// Finalized utterances, plus the live (replace-semantics) partial appended
// when painting.
private finalizedText = "";
private liveText = "";
private saveNotice = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private actions: LayerActions | null = null;
start(ctx: LayerContext): void {
this.actions = ctx.actions;
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.onTranscript(event);
ctx.actions.requestRender();
});
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
ctx.actions.requestRender();
});
void ctx.actions.setTranscribeRenderActive(true);
void Promise.resolve(ctx.actions.startVoiceCapture());
}
paint(): GrayImage {
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const text = this.displayText() || "Listening...";
const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64);
const wrapped = wrapTranscribeText(font, text, width - 64);
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
image.drawText(font, 24, 24, "Transcribe", 200);
image.drawText(font, 24, 46, this.status, 110);
image.drawRect(12, 12, width - 24, height - 24, 52);
image.drawText(font, 24, 20, "Transcribe", 200);
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
const firstLine = Math.max(0, wrapped.length - 10);
const bodyTop = 62;
const footerY = height - 18;
const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / 16));
const firstLine = Math.max(0, wrapped.length - bodyLines);
for (let index = firstLine; index < wrapped.length; index++) {
const y = 72 + (index - firstLine) * 16;
const y = bodyTop + (index - firstLine) * 16;
image.drawText(font, 32, y, wrapped[index]!, 230);
}
image.drawText(font, 24, 252, "Double-click: back", 110);
image.drawText(font, 24, footerY, "Click: save Double-click: back", 110);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "click") {
this.saveTranscript();
ctx.actions.requestRender();
return;
}
if (event.type === "double-click") {
ctx.stack.pop();
}
@@ -55,9 +70,17 @@ export class TranscribeLayer implements Layer {
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
void this.actions?.setTranscribeRenderActive(false);
void this.actions?.stopVoiceCapture();
this.actions = null;
}
private saveTranscript(): void {
const text = this.displayText().trim();
if (!text) {
this.saveNotice = "Nothing to save yet.";
return;
}
const filename = `transcript-${transcriptTimestamp()}.txt`;
const path = writeTextToDownloads(filename, `${text}\n`);
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
}
private displayText(): string {
@@ -80,6 +103,15 @@ export class TranscribeLayer implements Layer {
}
}
function transcriptTimestamp(): string {
const now = new Date();
const pad = (value: number) => String(value).padStart(2, "0");
return (
`${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}` +
`-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`
);
}
function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
@@ -98,4 +130,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri
}
return lines.length ? lines : [""];
}
-45
View File
@@ -1,45 +0,0 @@
import { type GrayImage } from "../graphics/image";
import { type MediaControllerState } from "../native/media-controller";
import { type NightscoutState } from "../native/nightscout-bridge";
import { type DashboardPluginId } from "./dashboard-settings";
import { type Layer } from "./layers";
export type DashboardPluginState = {
logLines: string[];
media: MediaControllerState;
nightscout: NightscoutState;
nightscoutConfigured: boolean;
};
export type DashboardPluginCardBounds = {
x: number;
y: number;
width: number;
height: number;
};
export type DashboardPluginCardRenderArgs = {
image: GrayImage;
bounds: DashboardPluginCardBounds;
selected: boolean;
state: DashboardPluginState;
};
export abstract class DashboardPlugin {
readonly id: DashboardPluginId;
readonly label: string;
constructor(config: {
id: DashboardPluginId;
label: string;
}) {
this.id = config.id;
this.label = config.label;
}
abstract renderCard(args: DashboardPluginCardRenderArgs): void;
createFullscreenLayer(getState: () => DashboardPluginState): Layer | null {
return null;
}
}
-41
View File
@@ -1,41 +0,0 @@
import { type DashboardPluginId } from "./dashboard-settings";
import { DashboardPlugin } from "./dashboard-plugin";
import { NightscoutDashboardPlugin } from "./apps/nightscout";
import { MusicControllerDashboardPlugin } from "./apps/music-controller";
import { DebugLogDashboardPlugin } from "./apps/debug-log";
export { DashboardPlugin } from "./dashboard-plugin";
export type { DashboardPluginCardBounds, DashboardPluginCardRenderArgs, DashboardPluginState } from "./dashboard-plugin";
let dashboardPlugins: DashboardPlugin[] | null = null;
function getDashboardPlugins(): DashboardPlugin[] {
if (!dashboardPlugins) {
dashboardPlugins = [
{
id: "blank",
label: "Blank",
renderCard: () => {},
createFullscreenLayer: () => null,
},
new DebugLogDashboardPlugin(),
new MusicControllerDashboardPlugin(),
new NightscoutDashboardPlugin(),
];
}
return dashboardPlugins;
};
export function listDashboardPlugins(): DashboardPlugin[] {
return [...getDashboardPlugins()];
}
export function getDashboardPlugin(pluginId: DashboardPluginId): DashboardPlugin {
const PLUGIN_MAP = new Map<DashboardPluginId, DashboardPlugin>(
getDashboardPlugins().map((plugin) => [plugin.id, plugin]),
);
return PLUGIN_MAP.get(pluginId) ?? PLUGIN_MAP.get("blank")!;
}
export function isBlankDashboardPlugin(pluginId: DashboardPluginId): boolean {
return pluginId === "blank";
}
-79
View File
@@ -11,24 +11,11 @@ import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
import { GrayImage } from "~/graphics/image";
export type DashboardSlotId = "bottom-left" | "bottom-right";
export const dashboardSlotIds: DashboardSlotId[] = ["bottom-left", "bottom-right"];
export type DashboardPluginId = "blank" | "input-debug-log" | "music-controller" | "nightscout";
export type DashboardSlotConfig = Record<DashboardSlotId, DashboardPluginId>;
export type NightscoutSettings = {
siteUrl: string;
apiToken: string;
};
export type BatteryDisplayMode = "icon" | "percentage";
export type SystemCardSettings = {
showFaceclawLogo: boolean;
showBatteryIndicators: boolean;
batteryDisplayMode: BatteryDisplayMode;
showAndroidNotifications: boolean;
showSignalStrength: boolean;
};
export type SystemCardSettingKey = keyof SystemCardSettings;
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "never";
type ConfigSettingOptions<TValue, TId extends string> = {
@@ -171,35 +158,6 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
}
const KNOWN_PLUGIN_IDS = new Set<DashboardPluginId>([
"blank",
"input-debug-log",
"music-controller",
"nightscout",
]);
export const bottomLeftSlotSetting = new ConfigSettingEnum<DashboardPluginId>({
id: "bottom-left-slot",
label: "Bottom left slot",
storageKey: "dashboard.slot.bottomLeft",
defaultValue: "blank",
values: Array.from(KNOWN_PLUGIN_IDS),
});
export const bottomRightSlotSetting = new ConfigSettingEnum<DashboardPluginId>({
id: "bottom-right-slot",
label: "Bottom right slot",
storageKey: "dashboard.slot.bottomRight",
defaultValue: "blank",
values: Array.from(KNOWN_PLUGIN_IDS),
});
export const dashboardSlotSettings: Record<DashboardSlotId, ConfigSettingEnum<DashboardPluginId>> = {
"bottom-left": bottomLeftSlotSetting,
"bottom-right": bottomRightSlotSetting,
};
export const systemCardNameSetting = new ConfigSettingString({
id: "dashboard-name",
label: "Dashboard name",
@@ -211,20 +169,6 @@ export const systemCardNameSetting = new ConfigSettingString({
formatValue: (value) => value || "Faceclaw",
});
export const showFaceclawLogoSetting = new ConfigSettingBoolean({
id: "showFaceclawLogo",
label: "Show Faceclaw Logo",
storageKey: "dashboard.systemCard.showFaceclawLogo",
defaultValue: true,
});
export const showBatteryIndicatorsSetting = new ConfigSettingBoolean({
id: "showBatteryIndicators",
label: "Show Battery Indicators",
storageKey: "dashboard.systemCard.showBatteryIndicators",
defaultValue: true,
});
export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMode>({
id: "batteryDisplayMode",
label: "Battery display",
@@ -234,20 +178,6 @@ export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMod
formatValue: batteryDisplayModeLabel,
});
export const showAndroidNotificationsSetting = new ConfigSettingBoolean({
id: "showAndroidNotifications",
label: "Show Android Notifications",
storageKey: "dashboard.systemCard.showAndroidNotifications",
defaultValue: true,
});
export const showSignalStrengthSetting = new ConfigSettingBoolean({
id: "showSignalStrength",
label: "Show Signal Strength",
storageKey: "dashboard.systemCard.showSignalStrength",
defaultValue: true,
});
export const screenTimeoutSetting = new ConfigSettingEnum<ScreenTimeoutSetting>({
id: "screen-timeout",
label: "Screen timeout",
@@ -388,15 +318,6 @@ export function isNightscoutSettingsConfigured(): boolean {
return nightscoutSiteUrlSetting.get().length > 0 && nightscoutApiTokenSetting.get().length > 0;
}
export function normalizePluginId(
value: string | null | undefined,
fallback: DashboardPluginId = "blank",
): DashboardPluginId {
if (value && KNOWN_PLUGIN_IDS.has(value as DashboardPluginId)) {
return value as DashboardPluginId;
}
return fallback;
}
function normalizeSystemCardName(name: string | null | undefined): string {
const normalized = (name ?? "").replace(/[\x00-\x1f]+/g, " ").replace(/\s+/g, " ").trim();
-170
View File
@@ -1,170 +0,0 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
import { getDashboardPlugin, type DashboardPluginCardBounds, type DashboardPluginState } from "./dashboard-plugins";
import { EditTextSettingLayer, isNightscoutSettingsConfigured, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings";
import { Layer, LayerActions, LayerStack, type DashboardInputEvent } from "./layers";
import { drawSystemCard } from "./dashboard/system-card";
import { createRootMenuLayer } from "./dashboard/root-menu";
import { inputEventToString, shell } from "./shell/shell";
type DashboardCardId = "system" | DashboardSlotId;
export type DashboardBatteryLevels = {
headset: number | null;
headsetCharging: boolean | null;
ring: number | null;
};
type DashboardState = {
logLines: string[];
battery: DashboardBatteryLevels;
};
export let dashboardState: DashboardState = {
logLines: [] as string[],
battery: {
headset: null,
headsetCharging: null,
ring: null,
} as DashboardBatteryLevels,
};
const dashboardActions: LayerActions = {
requestRender: () => {},
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
setTranscribeRenderActive: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
playBuzzerSequence: () => {},
};
export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 };
/**
* Input handler for the dashboard window. Global gestures (wake, sleep,
* long-press/push-to-talk) are consumed by the shell before this is called.
*/
export async function receiveDashboardWindowInput(inputEvent: DashboardInputEvent): Promise<void> {
dashboardState.logLines.push(inputEventToString(inputEvent));
await dashboardLayers.handleInput(inputEvent);
}
export function logToDashboard(message: string): void {
dashboardState.logLines.push(message);
}
export function setDashboardActions(actions: Partial<LayerActions>): void {
dashboardLayers.setActions(actions);
}
/**
* Close the glasses-side text-setting edit page, e.g. when the user finishes
* typing on the phone. Returns whether the display changed.
*/
export function closeDashboardTextSettingEditor(): boolean {
return dashboardLayers.popIfTop((layer) => layer instanceof EditTextSettingLayer);
}
export function drawDashboard(): GrayImage {
if (!shell.isScreenOn()) {
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
}
return dashboardLayers.paint();
}
export function resetDashboardSleepTimerAndWake(nowMs = Date.now()): boolean {
if (!shell.wake("sidebar", nowMs)) return false;
dashboardLayers.clearToBase();
dashboardLayers.push(createRootMenuLayer());
return true;
}
export function setDashboardBatteryLevels(levels: Partial<DashboardBatteryLevels>): void {
dashboardState.battery = {
...dashboardState.battery,
...levels,
};
}
export function getPluginIdForSlot(slot: DashboardSlotId): DashboardPluginId {
switch (slot) {
case "bottom-left":
return bottomLeftSlotSetting.get();
case "bottom-right":
return bottomRightSlotSetting.get();
}
}
class DashboardLayer implements Layer {
paint(): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const pluginState = getPluginState();
drawSystemCard(image, getCardBounds("system"));
for (const slot of dashboardSlotIds) {
const bounds = getCardBounds(slot);
const pluginId = getPluginIdForSlot(slot);
getDashboardPlugin(pluginId).renderCard({
image,
bounds,
selected: false,
state: pluginState,
});
}
return image;
}
handleInput(event: DashboardInputEvent, ctx: { stack: LayerStack }): void {
if (event.type === "click") {
ctx.stack.push(createRootMenuLayer());
return;
}
if (event.type === "double-click") {
// Backing out of the window's root returns focus to the shell sidebar.
shell.yieldFocusToSidebar();
}
}
}
export function getPluginState(): DashboardPluginState {
return {
logLines: dashboardState.logLines,
media: mediaControllerBridge.snapshot(),
nightscout: nightscoutBridge.snapshot(),
nightscoutConfigured: isNightscoutSettingsConfigured(),
};
}
function getCardBounds(card: DashboardCardId): DashboardPluginCardBounds {
switch (card) {
case "system":
return {
x: G2_LENS_WIDTH / 2 + 2,
y: 0,
width: G2_LENS_WIDTH / 2 - 2,
height: G2_LENS_HEIGHT / 2 - 2,
};
case "bottom-left":
return {
x: 0,
y: G2_LENS_HEIGHT / 2 + 2,
width: G2_LENS_WIDTH / 2 - 2,
height: G2_LENS_HEIGHT / 2 - 2,
};
case "bottom-right":
return {
x: G2_LENS_WIDTH / 2 + 2,
y: G2_LENS_HEIGHT / 2 + 2,
width: G2_LENS_WIDTH / 2 - 2,
height: G2_LENS_HEIGHT / 2 - 2,
};
}
}
const dashboardLayers = new LayerStack(new DashboardLayer(), dashboardActions);
dashboardLayers.push(createRootMenuLayer());
-86
View File
@@ -1,86 +0,0 @@
import { getDefaultSmallFont } from "~/graphics/bdffont";
import { getPluginIdForSlot, getPluginState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { getDashboardPlugin, isBlankDashboardPlugin } from "../dashboard-plugins";
import { DashboardSlotId, dashboardSlotIds, isNightscoutSettingsConfigured } from "../dashboard-settings";
import { DashboardInputEvent, LayerContext } from "../layers";
import { MenuLayer } from "../menu";
import { createNightscoutSettingsMenuLayer } from "./settings-menus";
import { TranscribeLayer } from "../apps/transcribe";
import { GrayImage } from "~/graphics/image";
import { shell } from "../shell/shell";
const rootMenuFont = getDefaultSmallFont();
class RootMenuLayer extends MenuLayer {
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
// Backing out of the window's root returns focus to the shell sidebar.
shell.yieldFocusToSidebar();
return;
}
await super.handleInput(event, ctx);
}
}
export function createRootMenuLayer(): MenuLayer {
return new RootMenuLayer(
null,
[
{
label: "Apps",
onSelect: (ctx) => {
ctx.stack.push(createAppsMenuLayer());
},
},
...dashboardSlotIds
.filter((slot) => !isBlankDashboardPlugin(getPluginIdForSlot(slot)))
.map((slot) => ({
label: pluginLabelForSlot(slot),
onSelect: (ctx: LayerContext) => {
openDashboardSlot(slot, ctx);
},
render: ({ image, x, y }: { image: GrayImage; x: number; y: number }) => {
image.drawText(rootMenuFont, x, y + 3, pluginLabelForSlot(slot), 200);
},
})),
],
TOP_LEFT_MENU_LAYOUT,
);
}
function createAppsMenuLayer(): MenuLayer {
return new MenuLayer(
"Apps",
[
{
label: "Transcribe",
onSelect: (ctx) => {
const layer = new TranscribeLayer();
ctx.stack.push(layer);
layer.start(ctx);
},
},
],
TOP_LEFT_MENU_LAYOUT,
);
}
function openDashboardSlot(slot: DashboardSlotId, ctx: LayerContext): void {
const pluginId = getPluginIdForSlot(slot);
if (isBlankDashboardPlugin(pluginId)) {
return;
}
if (pluginId === "nightscout" && !isNightscoutSettingsConfigured()) {
ctx.stack.push(createNightscoutSettingsMenuLayer());
return;
}
const layer = getDashboardPlugin(pluginId).createFullscreenLayer?.(getPluginState);
if (layer) {
ctx.stack.push(layer);
}
}
function pluginLabelForSlot(slot: DashboardSlotId): string {
return getDashboardPlugin(getPluginIdForSlot(slot)).label;
}
+8 -47
View File
@@ -1,14 +1,12 @@
import { APP_VIEWPORT } from "../shell/geometry";
import { type MenuLayout } from "../menu";
// Settings menus render in both the dashboard window (full lens) and the
// Settings app window (viewport-sized); cap heights to the smaller of the two.
const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 };
import { MenuLayer, type MenuLayout } from "../menu";
import { shell } from "../shell/shell";
import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { MenuLayer } from "../menu";
import { batteryDisplayModeSetting, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { AboutPage } from "./about-page";
// Rendered in the Settings app window (viewport-sized); cap height to it.
const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 };
export function createSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings",
@@ -19,12 +17,6 @@ export function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createDisplaySettingsMenuLayer());
},
},
{
label: "Dashboard",
onSelect: (ctx) => {
ctx.stack.push(createDashboardSettingsMenuLayer());
},
},
{
label: "Voice",
onSelect: (ctx) => {
@@ -96,10 +88,12 @@ function createDisplaySettingsMenuLayer(): MenuLayer {
"Settings > Display",
[
enumSettingMenuItem(screenTimeoutSetting, {
onChange: (ctx, newValue, oldValue) => {
onChange: () => {
shell.noteUserActivity();
},
}),
// Controls the top-bar battery indicators (icon vs percentage).
enumSettingMenuItem(batteryDisplayModeSetting),
],
SETTINGS_MENU_LAYOUT,
);
@@ -116,39 +110,6 @@ function createVoiceSettingsMenuLayer(): MenuLayer {
);
}
function createDashboardSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings > Dashboard",
[
{
label: "System Card",
onSelect: (ctx) => {
ctx.stack.push(createSystemCardSettingsMenuLayer());
},
},
...dashboardSlotIds.map((slot) => enumSettingMenuItem(dashboardSlotSettings[slot], {
style: "submenu",
})),
],
SETTINGS_MENU_LAYOUT,
);
}
function createSystemCardSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings > Dashboard > System Card",
[
textSettingMenuItem(systemCardNameSetting),
toggleSettingMenuItem(showFaceclawLogoSetting),
toggleSettingMenuItem(showBatteryIndicatorsSetting),
enumSettingMenuItem(batteryDisplayModeSetting),
toggleSettingMenuItem(showAndroidNotificationsSetting),
toggleSettingMenuItem(showSignalStrengthSetting),
],
SETTINGS_MENU_LAYOUT,
);
}
function createIntegrationsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings > Integrations",
-171
View File
@@ -1,171 +0,0 @@
import { GrayImage } from "../../graphics/image";
import { spanCurrent } from "../../native/frame-timings";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
import { BATTERY_ICON_WIDTH, drawBattery } from "../../graphics/battery";
import { readActiveNotificationIcons } from "../../native/notification-icons";
import { readPhoneBatteryState } from "../../native/phone-battery";
import { readSystemStatusIcons } from "../../native/system-status-icons";
import { type DashboardPluginCardBounds } from "../dashboard-plugins";
import { dashboardState } from "../dashboard";
import { getDefaultMediumFont, getDefaultSmallFont } from "~/graphics/bdffont";
import { getDashboardLogo } from "~/graphics/logo";
import { batteryDisplayModeSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, showSignalStrengthSetting, systemCardNameSetting } from "../dashboard-settings";
const NOTIFICATION_ICON_SIZE = 24;
const SYSTEM_CARD_ITEM_HEIGHT = 38;
const SYSTEM_CARD_ITEM_GAP = 2;
const BATTERY_ITEM_Y_OFFSET = 4;
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
type SystemCardFlowItem =
| { type: "battery"; label: string; percentCharge: number; isCharging: boolean }
| { type: "notification"; icon: GrayImage };
const smallFont = getDefaultSmallFont();
const mediumFont = getDefaultMediumFont();
function drawSystemCardFlowItems(image: GrayImage, bounds: DashboardPluginCardBounds): void {
const left = bounds.x + 10;
const top = bounds.y + 84;
const right = bounds.x + bounds.width - 10;
const bottom = bounds.y + bounds.height - 6;
const items: SystemCardFlowItem[] = [];
if (showAndroidNotificationsSetting.get()) {
const maxNotificationIcons = Math.max(0, ((right - left) / Math.max(1, NOTIFICATION_ICON_SIZE + SYSTEM_CARD_ITEM_GAP)) | 0) * 2;
const { icons, stale } = readActiveNotificationIcons(maxNotificationIcons, renderPassAllowsStaleData());
if (stale) {
noteStaleDataUsed();
}
for (const icon of icons) {
items.push({ type: "notification", icon });
}
}
items.push(...collectBatteryItems());
if (showSignalStrengthSetting.get()) {
for (const icon of readSystemStatusIcons()) {
items.push({ type: "notification", icon });
}
}
let itemX = left;
let itemY = top;
for (const item of items) {
const itemWidth = systemCardFlowItemWidth(item);
if (itemX > left && itemX + itemWidth > right) {
itemX = left;
itemY += SYSTEM_CARD_ITEM_HEIGHT + SYSTEM_CARD_ITEM_GAP;
}
if (itemY + SYSTEM_CARD_ITEM_HEIGHT > bottom) {
break;
}
if (item.type === "battery") {
drawBatteryFlowItem(image, itemX, itemY, item);
} else {
image.bitBlt(
item.icon,
itemX,
itemY + ((SYSTEM_CARD_ITEM_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0,
);
}
itemX += itemWidth + SYSTEM_CARD_ITEM_GAP;
}
}
function systemCardFlowItemWidth(item: SystemCardFlowItem): number {
if (item.type === "notification") {
return NOTIFICATION_ICON_SIZE;
}
if (batteryDisplayModeSetting.get() === "percentage") {
const percentText = formatBatteryPercent(item.percentCharge);
return Math.max(smallFont.measureText(percentText), smallFont.measureText(item.label));
}
return Math.max(BATTERY_ICON_WIDTH, smallFont.measureText(item.label));
}
function collectBatteryItems(): SystemCardFlowItem[] {
if (!showBatteryIndicatorsSetting.get()) return [];
const phone = readPhoneBatteryState();
const items: SystemCardFlowItem[] = [];
addBatteryItem(items, "Phone", phone.battery, phone.charging);
addBatteryItem(items, "G2", dashboardState.battery.headset, dashboardState.battery.headsetCharging);
addBatteryItem(items, "R1", dashboardState.battery.ring, null);
return items;
}
function addBatteryItem(
items: SystemCardFlowItem[],
label: string,
percentCharge: number | null,
isCharging: boolean | null,
): void {
if (percentCharge === null || !Number.isFinite(percentCharge)) return;
items.push({
type: "battery",
label,
percentCharge,
isCharging: Boolean(isCharging),
});
}
function formatBatteryPercent(percentCharge: number): string {
const clamped = Math.max(0, Math.min(100, Math.round(percentCharge)));
return `${clamped}%`;
}
function drawBatteryFlowItem(image: GrayImage, x: number, y: number, item: Extract<SystemCardFlowItem, { type: "battery" }>): void {
const itemWidth = systemCardFlowItemWidth(item);
const labelX = x + Math.max(0, ((itemWidth - smallFont.measureText(item.label)) / 2) | 0);
image.drawText(smallFont, labelX, y + BATTERY_ITEM_Y_OFFSET, item.label, 150);
if (batteryDisplayModeSetting.get() === "percentage") {
drawBatteryPercentFlowItem(image, x, y, item, itemWidth);
return;
}
const battery = drawBattery(item.percentCharge, item.isCharging);
image.bitBlt(battery, x + ((itemWidth - battery.width) / 2) | 0, y + 16 + BATTERY_ITEM_Y_OFFSET);
}
function drawBatteryPercentFlowItem(
image: GrayImage,
x: number,
y: number,
item: Extract<SystemCardFlowItem, { type: "battery" }>,
itemWidth: number,
): void {
const percentText = formatBatteryPercent(item.percentCharge);
const textWidth = smallFont.measureText(percentText);
const textX = x + Math.max(0, ((itemWidth - textWidth) / 2) | 0);
const textY = y + 16 + BATTERY_ITEM_Y_OFFSET;
if (item.isCharging) {
const padX = 2;
const padY = 1;
image.fillRect(textX - padX, textY - padY, textWidth + padX * 2, smallFont.lineHeight + padY * 2, 255);
image.drawText(smallFont, textX, textY, percentText, 0);
return;
}
image.drawText(smallFont, textX, textY, percentText, 200);
}
function formatDashboardDate(now: Date): string {
return `${WEEKDAYS[now.getDay()]} ${MONTHS[now.getMonth()]} ${now.getDate()} ${pad2(now.getHours())}:${pad2(now.getMinutes())}`;
}
function pad2(value: number): string {
return String(value).padStart(2, "0");
}
export function drawSystemCard(image: GrayImage, bounds: DashboardPluginCardBounds): void {
spanCurrent("draw-system-card", () => {
const now = new Date();
const logo = getDashboardLogo();
if (logo && showFaceclawLogoSetting.get()) {
image.bitBlt(logo, bounds.x + 10, bounds.y + 10);
}
const infoX = logo && showFaceclawLogoSetting.get() ? bounds.x + 92 : bounds.x + 10;
image.drawText(mediumFont, infoX, bounds.y + 10, systemCardNameSetting.get(), 200);
image.drawText(mediumFont, infoX, bounds.y + 32, formatDashboardDate(now), 200);
drawSystemCardFlowItems(image, bounds);
});
}
+5 -3
View File
@@ -18,11 +18,13 @@ export type LayerActions = {
disconnect: () => Promise<void> | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
/** Start voice capture with the provider chosen in settings. */
/** Start push-to-talk voice capture with the provider chosen in settings. */
startVoiceCapture: () => Promise<void> | void;
/** Stop voice capture; for a cloud provider this also commits for a final result. */
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
stopVoiceCapture: () => Promise<void> | void;
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
startContinuousVoiceCapture: () => Promise<void> | void;
stopContinuousVoiceCapture: () => Promise<void> | void;
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
};
+1 -2
View File
@@ -1,7 +1,6 @@
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { GrayImage } from "../../graphics/image";
import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge";
import { getPluginState } from "../dashboard";
import { NightscoutLayer } from "../apps/nightscout";
import {
createInProcessWindow,
@@ -35,7 +34,7 @@ export function createNightscoutAppWindow(options: InProcessAppOptions): InProce
iconLetter: "Ns",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(new NightscoutLayer(getPluginState)),
baseLayer: new YieldAtRootLayer(new NightscoutLayer()),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
+2 -1
View File
@@ -66,9 +66,10 @@ const noopActions: LayerActions = {
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
setTranscribeRenderActive: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
startContinuousVoiceCapture: () => {},
stopContinuousVoiceCapture: () => {},
playBuzzerSequence: () => {},
};
+77
View File
@@ -0,0 +1,77 @@
import { imageFromAsciiArt } from "../../graphics/image";
import { TranscribeLayer } from "../apps/transcribe";
import type { LayerActions } from "../layers";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "./in-process-window";
import { shell } from "./shell";
export const TRANSCRIBE_WINDOW_ID = "transcribe";
export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
const TRAY_ICON_ID = "transcribe";
// 16x18 microphone glyph for the top-bar tray.
const MIC_ICON = imageFromAsciiArt(
[
" #### ",
" ###### ",
" # # ",
" # # ",
" # # ",
" # # ",
" # # ",
" # # # # ",
" # # # # ",
" # #### # ",
" # # ",
" ###### ",
" # ",
" # ",
" ###### ",
],
220,
);
export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */
startContinuousVoiceCapture: () => void;
stopContinuousVoiceCapture: () => void;
};
/**
* The Transcribe app: live speech-to-text in its own window. While open it
* holds continuous mic capture (so push-to-talk can share the stream) and
* shows a microphone tray icon in the top bar.
*/
export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow {
const layer = new TranscribeLayer();
const actions: LayerActions = { ...options.actions };
const app = createInProcessWindow({
appId: "transcribe",
windowId: TRANSCRIBE_WINDOW_ID,
title: "Transcribe",
iconLetter: "Tr",
closeable: true,
actions,
baseLayer: new YieldAtRootLayer(layer),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
},
});
// start() wires the layer's transcript/status subscriptions; the LayerStack
// exposes its context through a paint, but the layer needs it up front, so
// build a minimal context here.
layer.start({ stack: app.stack, actions });
options.startContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
return app;
}