forked from Blendman974/faceclaw
Port Transcribe to new UI; remove legacy dashboard
This commit is contained in:
1 parent
56d0f8c351
commit
d0c30f9ca9
21 files changed
+392
-1291
No files matched your search
@@ -268,6 +268,31 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
compositor.configureScreen(width, height);
|
||||
}
|
||||
|
||||
/**
|
||||
* The current composited screen as a phone-UI preview bitmap, or null
|
||||
* before any surface has been configured. Built from the compositor so
|
||||
* the preview reflects every surface (chrome + whichever app is
|
||||
* foreground), including worker-app frames the TS side never sees.
|
||||
*/
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return null;
|
||||
}
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
|
||||
}
|
||||
|
||||
/** Save the current composite as a raw 4bpp screenshot; returns the path or "". */
|
||||
public String saveRawCompositeScreenshot() throws java.io.IOException {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return "";
|
||||
}
|
||||
return ScreenshotUtil.saveRawScreenshot(
|
||||
appContext, java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height);
|
||||
}
|
||||
|
||||
/** Show or hide a compositor surface; takes effect at the next composite. */
|
||||
public void setSurfaceVisible(String id, boolean visible) {
|
||||
compositor.setSurfaceVisible(id, visible);
|
||||
|
||||
@@ -217,6 +217,38 @@ public final class SurfaceCompositor {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Current composited pixels for the phone-side preview / screenshot, or
|
||||
* null before the screen is configured. Does not consume a sequence
|
||||
* number (it is never stored as the desired frame).
|
||||
*/
|
||||
public Composite previewComposite() {
|
||||
synchronized (lock) {
|
||||
if (screenWidth <= 0 || screenHeight <= 0) {
|
||||
return null;
|
||||
}
|
||||
byte[] gray = buildGrayLocked();
|
||||
return new Composite(gray, screenWidth, screenHeight, "preview", 0);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] buildGrayLocked() {
|
||||
byte[] gray = new byte[screenWidth * screenHeight];
|
||||
if (blanked) {
|
||||
return gray;
|
||||
}
|
||||
List<Surface> ordered = new ArrayList<>(surfaces.values());
|
||||
ordered.sort(Comparator
|
||||
.comparingInt((Surface s) -> s.zOrder)
|
||||
.thenComparing(s -> s.id));
|
||||
for (Surface surface : ordered) {
|
||||
if (surface.visible) {
|
||||
blendLocked(gray, surface);
|
||||
}
|
||||
}
|
||||
return gray;
|
||||
}
|
||||
|
||||
private Composite compositeLocked() {
|
||||
byte[] gray = new byte[screenWidth * screenHeight];
|
||||
if (blanked) {
|
||||
|
||||
+95
-404
@@ -14,17 +14,7 @@ import { firmwareIncompatibilityMessage } from "./firmware-compat";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import {
|
||||
closeDashboardTextSettingEditor,
|
||||
drawDashboard,
|
||||
logToDashboard,
|
||||
receiveDashboardWindowInput,
|
||||
resetDashboardSleepTimerAndWake,
|
||||
setDashboardBatteryLevels,
|
||||
setDashboardActions,
|
||||
} from "../ui/dashboard";
|
||||
import { inputEventToString, rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { makeLetterWindowIcon } from "../ui/shell/chrome-layer";
|
||||
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { WorkerAppHost } from "../ui/shell/worker-window";
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app";
|
||||
import {
|
||||
@@ -55,6 +45,11 @@ import {
|
||||
NIGHTSCOUT_WINDOW_ID,
|
||||
} from "../ui/shell/nightscout-app";
|
||||
import { createMusicAppWindow, MUSIC_SURFACE_ID, MUSIC_WINDOW_ID } from "../ui/shell/music-app";
|
||||
import {
|
||||
createTranscribeAppWindow,
|
||||
TRANSCRIBE_SURFACE_ID,
|
||||
TRANSCRIBE_WINDOW_ID,
|
||||
} from "../ui/shell/transcribe-app";
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { APP_VIEWPORT } from "../ui/shell/geometry";
|
||||
import { type LayerActions } from "../ui/layers";
|
||||
@@ -72,7 +67,6 @@ import {
|
||||
voiceProviderSetting,
|
||||
type ConfigSettingString,
|
||||
} from "../ui/dashboard-settings";
|
||||
import { saveRawScreenshot } from "../native/raw-screenshot";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
|
||||
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
|
||||
@@ -94,17 +88,15 @@ export type DashboardSnapshot = {
|
||||
};
|
||||
|
||||
type DashboardListener = (snapshot: DashboardSnapshot) => void;
|
||||
type LogLevel = "debug"|"info"|"warn"|"error";
|
||||
|
||||
const CONTAINER_NAME = "dashboard";
|
||||
// Compositor surfaces: the dashboard window in the app viewport, and the
|
||||
// shell chrome (sidebar + top bar + overlays) above it with color-key
|
||||
// transparency.
|
||||
const DASHBOARD_WINDOW_SURFACE_ID = "window:dashboard";
|
||||
// The shell chrome (sidebar + top bar + overlays) composites above all app
|
||||
// window surfaces with color-key transparency.
|
||||
const SHELL_SURFACE_ID = "shell";
|
||||
const DASHBOARD_INTERVAL_MS = 60_000;
|
||||
// Top-bar clock refresh; the phone-side preview polls the Java composite so
|
||||
// it reflects every app (including worker apps the TS side never renders).
|
||||
const SHELL_REFRESH_INTERVAL_MS = 60_000;
|
||||
const PREVIEW_INTERVAL_MS = 1_000;
|
||||
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
|
||||
const TRANSCRIBE_RENDER_INTERVAL_MS = 250;
|
||||
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
|
||||
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
|
||||
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
|
||||
@@ -119,32 +111,6 @@ function formatTimestamp(date: Date): string {
|
||||
return date.toISOString().slice(11, 23);
|
||||
}
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function formatTime24h(date: Date, includeSeconds: boolean): string {
|
||||
const hours = date.getHours();
|
||||
const minutes = pad2(date.getMinutes());
|
||||
const seconds = pad2(date.getSeconds());
|
||||
return includeSeconds
|
||||
? `${hours}:${minutes}:${seconds}`
|
||||
: `${hours}:${minutes}`;
|
||||
}
|
||||
|
||||
function formatUtcOffset(date: Date): string {
|
||||
const offsetMinutes = -date.getTimezoneOffset();
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const absoluteMinutes = Math.abs(offsetMinutes);
|
||||
const hours = Math.floor(absoluteMinutes / 60);
|
||||
const minutes = absoluteMinutes % 60;
|
||||
return `UTC${sign}${pad2(hours)}:${pad2(minutes)}`;
|
||||
}
|
||||
|
||||
function formatForStatus(date: Date): string {
|
||||
return formatTime24h(date, true);
|
||||
}
|
||||
|
||||
function eventName(eventType: number): string {
|
||||
return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`;
|
||||
}
|
||||
@@ -153,25 +119,6 @@ function sourceName(eventSource: number): string {
|
||||
return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`;
|
||||
}
|
||||
|
||||
function normalizeContainerName(name: string): string {
|
||||
return name.replace(/[^\x20-\x7e]+/g, "");
|
||||
}
|
||||
|
||||
function isDashboardContainerName(name: string): boolean {
|
||||
const normalized = normalizeContainerName(name);
|
||||
return normalized === CONTAINER_NAME || normalized.includes(CONTAINER_NAME);
|
||||
}
|
||||
|
||||
/** Top-left viewport-sized crop of a full-lens dashboard paint. */
|
||||
function cropToViewport(image: GrayImage): GrayImage {
|
||||
if (image.width === APP_VIEWPORT.width && image.height === APP_VIEWPORT.height) {
|
||||
return image;
|
||||
}
|
||||
const cropped = new GrayImage(APP_VIEWPORT.width, APP_VIEWPORT.height, 0);
|
||||
cropped.bitBlt(image, 0, 0, { width: APP_VIEWPORT.width, height: APP_VIEWPORT.height });
|
||||
return cropped;
|
||||
}
|
||||
|
||||
class DashboardController {
|
||||
private phase: ConnectionPhase = "disconnected";
|
||||
private status = "Disconnected.";
|
||||
@@ -185,9 +132,9 @@ class DashboardController {
|
||||
private readonly listeners = new Set<DashboardListener>();
|
||||
|
||||
private communicator: FaceclawCommunicatorBridge | null = null;
|
||||
private dashboardTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private shellRefreshTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private previewTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private transcribeRenderTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private offState: (() => void) | null = null;
|
||||
private offLog: (() => void) | null = null;
|
||||
private offRing: (() => void) | null = null;
|
||||
@@ -195,23 +142,14 @@ class DashboardController {
|
||||
private offEvenAppConflict: (() => void) | null = null;
|
||||
private offFrameMetrics: (() => void) | null = null;
|
||||
private offFirmwareInfo: (() => void) | null = null;
|
||||
private offMedia: (() => void) | null = null;
|
||||
private offNightscout: (() => void) | null = null;
|
||||
private offVoiceStatus: (() => void) | null = null;
|
||||
private offVoiceWakeWord: (() => void) | null = null;
|
||||
private offAndroidNotification: (() => void) | null = null;
|
||||
private lastInput = "waiting...";
|
||||
private lastSys = "none yet";
|
||||
private renderInProgress = false;
|
||||
private renderQueued = false;
|
||||
private queuedRenderReason: "initial" | "interval" = "interval";
|
||||
private queuedFrameId = 0;
|
||||
private shellRenderInProgress = false;
|
||||
private shellRenderQueued = false;
|
||||
private nextShellRenderWantsFreshData = false;
|
||||
// Last painted surface images, for compositing the phone-side preview.
|
||||
private lastShellImage: GrayImage | null = null;
|
||||
private lastWindowImage: GrayImage | null = null;
|
||||
// One shared worker per app hosts all its windows; spawned on first launch.
|
||||
private readonly appHosts = new Map<string, WorkerAppHost>();
|
||||
private nextWindowSerial = 1;
|
||||
@@ -223,29 +161,19 @@ class DashboardController {
|
||||
private sharedActions!: Omit<LayerActions, "requestRender">;
|
||||
private lastForegroundNotificationUpdateAtMs = 0;
|
||||
private lastConnectedPreviewUpdateAtMs = 0;
|
||||
// Consumed by the next renderDashboard: repaint with data sources not
|
||||
// allowed to serve stale caches (set after a frame painted with stale data).
|
||||
private nextRenderWantsFreshData = false;
|
||||
|
||||
constructor() {
|
||||
const sharedActions = {
|
||||
disconnect: () => this.disconnect(),
|
||||
startTextSettingEdit: (setting: ConfigSettingString) => this.startTextSettingEdit(setting),
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
setTranscribeRenderActive: (active: boolean) => this.setTranscribeRenderActive(active),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
setDashboardActions({
|
||||
...sharedActions,
|
||||
requestRender: () => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`requested render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
},
|
||||
});
|
||||
shell.configure({
|
||||
actions: {
|
||||
...sharedActions,
|
||||
@@ -282,6 +210,7 @@ class DashboardController {
|
||||
{ appId: "teleprompt", label: "Teleprompt" },
|
||||
{ appId: "music", label: "Music" },
|
||||
{ appId: "nightscout", label: "Nightscout" },
|
||||
{ appId: "transcribe", label: "Transcribe" },
|
||||
{ appId: "notifications", label: "Notifications" },
|
||||
{ appId: "debug-tests", label: "Debug tests" },
|
||||
{ appId: "settings", label: "Settings" },
|
||||
@@ -294,26 +223,6 @@ class DashboardController {
|
||||
},
|
||||
}),
|
||||
);
|
||||
shell.registerWindow({
|
||||
appId: "dashboard",
|
||||
windowId: "dashboard",
|
||||
title: "Dashboard",
|
||||
surfaceId: DASHBOARD_WINDOW_SURFACE_ID,
|
||||
closeable: false,
|
||||
drawIcon: makeLetterWindowIcon("D"),
|
||||
handleInput: async (event, frameId) => {
|
||||
await receiveDashboardWindowInput(event);
|
||||
await this.requestRender("interval", frameId);
|
||||
},
|
||||
requestRender: () => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`window render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
},
|
||||
setForeground: (foreground) => {
|
||||
this.setWindowSurfaceVisible(DASHBOARD_WINDOW_SURFACE_ID, foreground);
|
||||
},
|
||||
});
|
||||
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => {
|
||||
void this.handleAndroidNotificationPosted(notificationKey).catch((error) => {
|
||||
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
|
||||
@@ -336,12 +245,12 @@ class DashboardController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the current dashboard as the raw headerless 4bpp frame buffer the
|
||||
* wire compressor sees; test data for compression experiments.
|
||||
* Save the current composited screen as the raw headerless 4bpp frame
|
||||
* buffer the wire compressor sees; test data for compression experiments.
|
||||
*/
|
||||
saveRawDashboardScreenshot(): string {
|
||||
const path = saveRawScreenshot(drawDashboard());
|
||||
this.appendLog(`raw screenshot saved: ${path}`);
|
||||
const path = this.communicator?.saveRawCompositeScreenshot() ?? "";
|
||||
this.appendLog(path ? `raw screenshot saved: ${path}` : "raw screenshot skipped: not connected");
|
||||
return path;
|
||||
}
|
||||
|
||||
@@ -433,7 +342,7 @@ class DashboardController {
|
||||
// truth while typing; echoing activeTextSettingValue back into its two-way
|
||||
// binding on every keystroke drops fast/pasted characters (observed: a
|
||||
// 51-char API key stored as its first 46 chars). Just refresh the preview.
|
||||
this.previewOrRenderAfterTextSettingChange(setting.label);
|
||||
this.previewOrRenderAfterTextSettingChange();
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
@@ -452,7 +361,6 @@ class DashboardController {
|
||||
this.lastSys = "none yet";
|
||||
this.firmwareWarningMessage = "";
|
||||
this.refreshBatteryOptimizationStatus();
|
||||
resetDashboardSleepTimerAndWake();
|
||||
this.refreshEvenAppStatus();
|
||||
this.setPhase("connecting");
|
||||
this.setStatus("Connecting to the glasses...");
|
||||
@@ -505,20 +413,13 @@ class DashboardController {
|
||||
});
|
||||
});
|
||||
this.offBattery = communicator.onBatteryState((state) => {
|
||||
setDashboardBatteryLevels({
|
||||
headset: state.battery,
|
||||
headsetCharging: state.chargingStatus > 0,
|
||||
});
|
||||
shell.setBatteryLevels({
|
||||
headset: state.battery,
|
||||
headsetCharging: state.chargingStatus > 0,
|
||||
});
|
||||
if ((this.phase === "connected" || this.phase === "charging") && this.communicator) {
|
||||
// Repaint the top bar (battery indicators live in the shell chrome).
|
||||
this.requestShellRender();
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
this.appendLog(`battery update failed: ${message}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
this.offEvenAppConflict = communicator.onEvenAppConflict((message) => {
|
||||
@@ -550,22 +451,8 @@ class DashboardController {
|
||||
this.emit();
|
||||
}
|
||||
});
|
||||
this.offMedia = mediaControllerBridge.onStateChange(() => {
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
this.appendLog(`media update failed: ${message}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
this.offNightscout = nightscoutBridge.onStateChange(() => {
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
this.appendLog(`nightscout update failed: ${message}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
// The Music and Nightscout apps subscribe to their bridges directly and
|
||||
// repaint their own windows, so bridge updates need no controller action.
|
||||
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.appendLog(state.status);
|
||||
});
|
||||
@@ -577,8 +464,8 @@ class DashboardController {
|
||||
|
||||
await mediaControllerBridge.start();
|
||||
await nightscoutBridge.start();
|
||||
// Register the compositor surfaces: the dashboard window in the app
|
||||
// viewport, and the shell chrome above it.
|
||||
// Register the compositor surfaces: the shell chrome above all windows,
|
||||
// and a surface per live window (only the foreground one is composited).
|
||||
await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
|
||||
await communicator.configureSurface(SHELL_SURFACE_ID, {
|
||||
x: 0,
|
||||
@@ -588,8 +475,6 @@ class DashboardController {
|
||||
zOrder: 1,
|
||||
transparency: "color-key",
|
||||
});
|
||||
// Window surfaces for every live window (including any launched while
|
||||
// disconnected); only the foreground window's surface is composited.
|
||||
const foregroundWindowId = shell.foregroundWindow()?.windowId;
|
||||
for (const window of shell.getWindows()) {
|
||||
await this.configureWindowSurface(window.surfaceId, window.windowId === foregroundWindowId);
|
||||
@@ -597,25 +482,19 @@ class DashboardController {
|
||||
await communicator.start();
|
||||
shell.foregroundWindow()?.requestRender();
|
||||
this.requestShellRender();
|
||||
await this.requestRender("initial");
|
||||
this.dashboardTimer = setInterval(() => {
|
||||
// Also refreshes the top-bar clock on the shell surface.
|
||||
// Refresh the top-bar clock and the phone-side preview once a minute,
|
||||
// and keep the Android persistent notification current.
|
||||
this.shellRefreshTimer = setInterval(() => {
|
||||
this.requestShellRender();
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
this.setStatus(`Dashboard update failed: ${message}`);
|
||||
this.appendLog(`dashboard update failed: ${message}`);
|
||||
});
|
||||
}, DASHBOARD_INTERVAL_MS);
|
||||
this.updateCompositePreview();
|
||||
this.updateConnectedForegroundNotification();
|
||||
}, SHELL_REFRESH_INTERVAL_MS);
|
||||
this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS);
|
||||
this.screenTimeoutTimer = setInterval(() => {
|
||||
if (this.phase !== "connected" || !this.communicator) return;
|
||||
if (!shell.applyScreenTimeout()) return;
|
||||
this.endTextSettingEdit();
|
||||
this.requestShellRender();
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
const message = this.formatError(error);
|
||||
this.appendLog(`screen timeout render failed: ${message}`);
|
||||
});
|
||||
}, SCREEN_TIMEOUT_CHECK_MS);
|
||||
} catch (error) {
|
||||
const message = this.formatError(error);
|
||||
@@ -633,18 +512,10 @@ class DashboardController {
|
||||
this.offFrameMetrics = null;
|
||||
this.offFirmwareInfo?.();
|
||||
this.offFirmwareInfo = null;
|
||||
this.offMedia?.();
|
||||
this.offMedia = null;
|
||||
this.offNightscout?.();
|
||||
this.offNightscout = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
this.offVoiceWakeWord = null;
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
@@ -681,10 +552,6 @@ class DashboardController {
|
||||
this.offFrameMetrics = null;
|
||||
this.offFirmwareInfo?.();
|
||||
this.offFirmwareInfo = null;
|
||||
this.offMedia?.();
|
||||
this.offMedia = null;
|
||||
this.offNightscout?.();
|
||||
this.offNightscout = null;
|
||||
this.offVoiceStatus?.();
|
||||
this.offVoiceStatus = null;
|
||||
this.offVoiceWakeWord?.();
|
||||
@@ -706,10 +573,6 @@ class DashboardController {
|
||||
await mediaControllerBridge.stop().catch(() => {});
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
voiceControlBridge.stop();
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
await communicator?.close().catch(() => {});
|
||||
} finally {
|
||||
stopForegroundNotification();
|
||||
@@ -754,6 +617,22 @@ class DashboardController {
|
||||
* gate even though the audio source is the G2 mic over BLE.
|
||||
*/
|
||||
private startVoiceCapture(): void {
|
||||
this.beginVoiceCapture("ptt");
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private startContinuousVoiceCapture(): void {
|
||||
this.beginVoiceCapture("continuous");
|
||||
}
|
||||
|
||||
private stopContinuousVoiceCapture(): void {
|
||||
voiceControlBridge.stopContinuousCapture();
|
||||
}
|
||||
|
||||
private beginVoiceCapture(kind: "ptt" | "continuous"): void {
|
||||
if (this.phase !== "connected" || !this.communicator) {
|
||||
return;
|
||||
}
|
||||
@@ -761,22 +640,23 @@ class DashboardController {
|
||||
void ensureVoicePermissions()
|
||||
.then(() => {
|
||||
if (this.phase !== "connected" || this.communicator !== communicator) return;
|
||||
voiceControlBridge.startPushToTalk({
|
||||
const options = {
|
||||
communicator: communicator.getNativeCommunicator(),
|
||||
provider: voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
});
|
||||
};
|
||||
if (kind === "ptt") {
|
||||
voiceControlBridge.startPushToTalk(options);
|
||||
} else {
|
||||
voiceControlBridge.startContinuousCapture(options);
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
this.appendLog(`voice permission failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private endTextSettingEdit(): void {
|
||||
const finishedSetting = this.activeTextSetting;
|
||||
this.activeTextSetting = null;
|
||||
@@ -788,23 +668,14 @@ class DashboardController {
|
||||
|
||||
/**
|
||||
* Finish the active edit from the phone side (e.g. the IME's done key):
|
||||
* ends the edit session and navigates the glasses out of the edit page,
|
||||
* in whichever window hosts it (the Settings app or the dashboard).
|
||||
* ends the edit session and navigates the Settings app's glasses editor
|
||||
* out of the edit page.
|
||||
*/
|
||||
finishActiveTextSettingEdit(): void {
|
||||
if (!this.activeTextSetting) return;
|
||||
this.endTextSettingEdit();
|
||||
if (this.settingsApp?.closeTextEditor()) {
|
||||
this.settingsApp.requestRender();
|
||||
return;
|
||||
}
|
||||
if (!closeDashboardTextSettingEditor()) return;
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`edit close render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
} else {
|
||||
this.updateDisplayPreviewFromImage(drawDashboard());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -812,7 +683,7 @@ class DashboardController {
|
||||
if (setting.get() !== value) {
|
||||
setting.set(value);
|
||||
this.emit();
|
||||
this.previewOrRenderAfterTextSettingChange(setting.label);
|
||||
this.previewOrRenderAfterTextSettingChange();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -820,151 +691,13 @@ class DashboardController {
|
||||
await nightscoutBridge.refreshNow().catch((error) => {
|
||||
this.appendLog(`nightscout settings refresh failed: ${this.formatError(error)}`);
|
||||
});
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
await this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`nightscout settings render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
} else {
|
||||
const image = drawDashboard();
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
}
|
||||
|
||||
private previewOrRenderAfterTextSettingChange(label: string): void {
|
||||
// Echo phone-side keystrokes into the glasses editor, wherever it lives.
|
||||
private previewOrRenderAfterTextSettingChange(): void {
|
||||
// Echo phone-side keystrokes into the Settings app's glasses editor.
|
||||
if (this.settingsApp?.isTextEditorOnTop()) {
|
||||
this.settingsApp.requestRender();
|
||||
return;
|
||||
}
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`${label} update failed: ${this.formatError(error)}`);
|
||||
});
|
||||
return;
|
||||
}
|
||||
const image = drawDashboard();
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize renders; frameId identifies the input event or timer tick that
|
||||
* asked for this render (one is started here for callers that don't have
|
||||
* one). Ownership of the frame passes to renderDashboard and then to the
|
||||
* Java side; requests that coalesce into an already-queued render finish
|
||||
* immediately as discarded. When coalescing we keep the oldest waiting frame
|
||||
* so measured latency reflects the worst-served request.
|
||||
*/
|
||||
private async requestRender(reason: "initial" | "interval", frameId?: number): Promise<void> {
|
||||
const newFrameId = frameId ?? frameTimings.startFrame(`render:${reason}`);
|
||||
this.queuedRenderReason = this.queuedRenderReason === "initial" ? "initial" : reason;
|
||||
if (this.renderInProgress) {
|
||||
if (this.renderQueued && this.queuedFrameId > 0) {
|
||||
frameTimings.finishFrame(newFrameId, `discarded: coalesced into frame#${this.queuedFrameId}`);
|
||||
} else {
|
||||
this.renderQueued = true;
|
||||
this.queuedFrameId = newFrameId;
|
||||
frameTimings.logFrame(newFrameId, "render queued behind in-progress render");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this.renderInProgress = true;
|
||||
let frameToRender = newFrameId;
|
||||
try {
|
||||
while (true) {
|
||||
const nextReason = this.queuedRenderReason;
|
||||
this.renderQueued = false;
|
||||
this.queuedRenderReason = "interval";
|
||||
try {
|
||||
await this.renderDashboard(nextReason, frameToRender);
|
||||
} catch (error) {
|
||||
frameTimings.finishFrame(frameToRender, `discarded: render failed: ${this.formatError(error)}`);
|
||||
throw error;
|
||||
}
|
||||
if (!this.renderQueued || this.queuedFrameId <= 0) break;
|
||||
frameToRender = this.queuedFrameId;
|
||||
this.queuedFrameId = 0;
|
||||
}
|
||||
} finally {
|
||||
this.renderInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async renderDashboard(reason: "initial" | "interval", frameId: number): Promise<void> {
|
||||
console.log("renderDashboard", reason);
|
||||
frameTimings.logFrame(frameId, `renderDashboard start (${reason})`);
|
||||
const wantFreshData = this.nextRenderWantsFreshData;
|
||||
this.nextRenderWantsFreshData = false;
|
||||
beginRenderPass(!wantFreshData);
|
||||
const paintStartedAtMs = Date.now();
|
||||
const image = frameTimings.span(frameId, "paint", () =>
|
||||
frameTimings.runWithFrame(frameId, () => drawDashboard()),
|
||||
);
|
||||
const paintMs = Date.now() - paintStartedAtMs;
|
||||
const paintUsedStaleData = endRenderPass();
|
||||
if (paintUsedStaleData) {
|
||||
frameTimings.logFrame(frameId, "painted with stale data; will schedule a fresh-data repaint");
|
||||
}
|
||||
// The dashboard still paints the full lens; its window surface only shows
|
||||
// the viewport-sized top-left crop (content near the right/bottom edges
|
||||
// clips until the legacy layouts are resized).
|
||||
const cropped = frameTimings.span(frameId, "crop", () => cropToViewport(image));
|
||||
const fingerprint = frameTimings.span(frameId, "fingerprint", () => cropped.fingerprint());
|
||||
this.lastWindowImage = cropped;
|
||||
const updatePreviewAfterTransmit = this.phase === "connected";
|
||||
if (!updatePreviewAfterTransmit) {
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
if (this.communicator && this.phase === "charging") {
|
||||
// Glasses are in the charging case; keep the phone preview fresh but
|
||||
// send nothing.
|
||||
frameTimings.finishFrame(frameId, "discarded: glasses charging");
|
||||
this.updateConnectedForegroundNotification();
|
||||
console.log("renderDashbaord finished (charging)");
|
||||
return;
|
||||
}
|
||||
if (this.communicator) {
|
||||
if (shell.isScreenOn()) {
|
||||
await this.communicator.setG2ScreenOn(true);
|
||||
}
|
||||
console.log("submitSurfaceFrame");
|
||||
const buffer = frameTimings.span(frameId, "to8bpp", () => cropped.to8bppBuffer());
|
||||
// The Java side owns the frame from here: it finishes it on last-packet
|
||||
// ack, dedup, supersede, or timeout.
|
||||
await this.communicator.submitSurfaceFrame(
|
||||
DASHBOARD_WINDOW_SURFACE_ID,
|
||||
buffer,
|
||||
{ x: 0, y: 0, width: cropped.width, height: cropped.height },
|
||||
fingerprint,
|
||||
paintMs,
|
||||
frameId,
|
||||
);
|
||||
const outcome = await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
|
||||
if (outcome === null) {
|
||||
frameTimings.logFrame(frameId, "transmit backpressure wait timed out; render loop continuing");
|
||||
}
|
||||
if (!shell.isScreenOn()) {
|
||||
await this.communicator.setG2ScreenOn(false);
|
||||
}
|
||||
if (updatePreviewAfterTransmit) {
|
||||
this.updateConnectedCompositePreview();
|
||||
}
|
||||
} else {
|
||||
frameTimings.finishFrame(frameId, "discarded: no communicator, preview only");
|
||||
}
|
||||
|
||||
if (paintUsedStaleData) {
|
||||
this.nextRenderWantsFreshData = true;
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`fresh-data repaint failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
this.updateConnectedForegroundNotification();
|
||||
if (reason === "initial") {
|
||||
this.appendLog("initial dashboard image queued");
|
||||
}
|
||||
console.log("renderDashboard finished");
|
||||
}
|
||||
|
||||
private async handleInputEvent(event: RawInputEvent): Promise<void> {
|
||||
@@ -974,7 +707,6 @@ class DashboardController {
|
||||
let frameOwned = false;
|
||||
try {
|
||||
const inputEvent = rawInputEventToInputEvent(event);
|
||||
logToDashboard(inputEventToString(inputEvent));
|
||||
frameTimings.spanStart(frameId, "handle-input");
|
||||
let outcome: ShellInputOutcome;
|
||||
try {
|
||||
@@ -988,7 +720,6 @@ class DashboardController {
|
||||
if (event.kind === "sys-event") {
|
||||
this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`;
|
||||
this.appendLog(`sys-event ${this.lastSys}`);
|
||||
|
||||
if (
|
||||
event.eventType === OsEventTypeList.FOREGROUND_EXIT_EVENT ||
|
||||
event.eventType === OsEventTypeList.ABNORMAL_EXIT_EVENT ||
|
||||
@@ -996,19 +727,9 @@ class DashboardController {
|
||||
) {
|
||||
this.appendLog("display state invalidated by firmware exit event");
|
||||
}
|
||||
|
||||
if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
|
||||
this.lastInput = eventName(event.eventType);
|
||||
}
|
||||
} else if (event.kind === "text-click" && isDashboardContainerName(event.containerName)) {
|
||||
if (
|
||||
event.eventType === OsEventTypeList.SCROLL_TOP_EVENT ||
|
||||
event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT
|
||||
) {
|
||||
this.lastSys = `TEXT/${eventName(event.eventType)}`;
|
||||
this.lastInput = `TEXT_${eventName(event.eventType)}`;
|
||||
this.appendLog(`text-event ${this.lastSys}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (outcome.shell) {
|
||||
@@ -1144,6 +865,16 @@ class DashboardController {
|
||||
await this.launchInProcessApp(MUSIC_WINDOW_ID, MUSIC_SURFACE_ID, createMusicAppWindow);
|
||||
return;
|
||||
}
|
||||
if (appId === "transcribe") {
|
||||
await this.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) =>
|
||||
createTranscribeAppWindow({
|
||||
...options,
|
||||
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (appId === "teleprompt") {
|
||||
await this.launchInProcessApp(TELEPROMPT_WINDOW_ID, TELEPROMPT_SURFACE_ID, (options) =>
|
||||
createTelepromptBrowserWindow({
|
||||
@@ -1268,7 +999,6 @@ class DashboardController {
|
||||
this.nextShellRenderWantsFreshData = true;
|
||||
this.requestShellRender();
|
||||
}
|
||||
this.lastShellImage = image;
|
||||
if (!this.communicator || this.phase === "charging") {
|
||||
frameTimings.finishFrame(frameId, "discarded: shell render with no active connection");
|
||||
return;
|
||||
@@ -1284,24 +1014,15 @@ class DashboardController {
|
||||
frameId,
|
||||
);
|
||||
await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
|
||||
this.updateConnectedCompositePreview();
|
||||
this.updateCompositePreview();
|
||||
}
|
||||
|
||||
private async handleWakeWord(keyword: string): Promise<void> {
|
||||
const normalized = keyword.trim();
|
||||
if (normalized.length === 0) return;
|
||||
this.appendLog(`wake-word detected: ${normalized}`);
|
||||
const changed = resetDashboardSleepTimerAndWake();
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
if (changed) {
|
||||
this.requestShellRender();
|
||||
await this.requestRender("initial");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (changed) {
|
||||
const image = drawDashboard();
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
if (shell.wake("sidebar")) {
|
||||
this.requestShellRender();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1321,9 +1042,6 @@ class DashboardController {
|
||||
}
|
||||
shell.openNotificationModal(notificationKey, wokeScreen);
|
||||
this.requestShellRender();
|
||||
if (this.phase === "connected" && this.communicator) {
|
||||
await this.requestRender("interval");
|
||||
}
|
||||
}
|
||||
|
||||
private async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
@@ -1333,35 +1051,19 @@ class DashboardController {
|
||||
await this.communicator.playBuzzerSequence(payload);
|
||||
}
|
||||
|
||||
private setTranscribeRenderActive(active: boolean): void {
|
||||
if (!active || this.phase !== "connected" || !this.communicator) {
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (this.transcribeRenderTimer) return;
|
||||
this.transcribeRenderTimer = setInterval(() => {
|
||||
void this.requestRender("interval").catch((error) => {
|
||||
this.appendLog(`transcribe render failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}, TRANSCRIBE_RENDER_INTERVAL_MS);
|
||||
}
|
||||
|
||||
private clearDashboardTimer(): void {
|
||||
if (this.dashboardTimer) {
|
||||
clearInterval(this.dashboardTimer);
|
||||
this.dashboardTimer = null;
|
||||
if (this.shellRefreshTimer) {
|
||||
clearInterval(this.shellRefreshTimer);
|
||||
this.shellRefreshTimer = null;
|
||||
}
|
||||
if (this.previewTimer) {
|
||||
clearInterval(this.previewTimer);
|
||||
this.previewTimer = null;
|
||||
}
|
||||
if (this.screenTimeoutTimer) {
|
||||
clearInterval(this.screenTimeoutTimer);
|
||||
this.screenTimeoutTimer = null;
|
||||
}
|
||||
if (this.transcribeRenderTimer) {
|
||||
clearInterval(this.transcribeRenderTimer);
|
||||
this.transcribeRenderTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private setPhase(phase: ConnectionPhase): void {
|
||||
@@ -1402,27 +1104,13 @@ class DashboardController {
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private updateDisplayPreviewFromImage(image: GrayImage): void {
|
||||
this.setDisplayPreview(grayImageToPreviewSource(image));
|
||||
}
|
||||
|
||||
/**
|
||||
* Phone-side preview of what is actually on the glasses: the window surface
|
||||
* in its viewport with the shell chrome composited over it, mirroring the
|
||||
* Java compositor.
|
||||
* Phone-side preview of what is on the glasses, fetched from the Java
|
||||
* compositor so it reflects every surface (chrome + whichever app is
|
||||
* foreground, including worker apps the TS side never renders). Throttled
|
||||
* to avoid rebuilding the bitmap faster than the phone UI needs it.
|
||||
*/
|
||||
private buildCompositePreviewImage(): GrayImage {
|
||||
const preview = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
if (this.lastWindowImage) {
|
||||
preview.bitBlt(this.lastWindowImage, APP_VIEWPORT.x, APP_VIEWPORT.y);
|
||||
}
|
||||
if (this.lastShellImage) {
|
||||
preview.bitBlt(this.lastShellImage, 0, 0, { transparentZero: true });
|
||||
}
|
||||
return preview;
|
||||
}
|
||||
|
||||
private updateConnectedCompositePreview(): void {
|
||||
private updateCompositePreview(): void {
|
||||
if (!this.communicator) return;
|
||||
const now = Date.now();
|
||||
if (
|
||||
@@ -1432,7 +1120,10 @@ class DashboardController {
|
||||
return;
|
||||
}
|
||||
this.lastConnectedPreviewUpdateAtMs = now;
|
||||
this.updateDisplayPreviewFromImage(this.buildCompositePreviewImage());
|
||||
const preview = this.communicator.getCompositePreview();
|
||||
if (preview) {
|
||||
this.setDisplayPreview(preview);
|
||||
}
|
||||
}
|
||||
|
||||
private formatError(error: unknown): string {
|
||||
@@ -1448,7 +1139,7 @@ class DashboardController {
|
||||
case "click":
|
||||
return {
|
||||
kind: "sys-event",
|
||||
containerName: CONTAINER_NAME,
|
||||
containerName: "",
|
||||
eventType: OsEventTypeList.CLICK_EVENT,
|
||||
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
|
||||
systemExitReasonCode: 0,
|
||||
@@ -1457,7 +1148,7 @@ class DashboardController {
|
||||
case "double-click":
|
||||
return {
|
||||
kind: "sys-event",
|
||||
containerName: CONTAINER_NAME,
|
||||
containerName: "",
|
||||
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
|
||||
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
|
||||
systemExitReasonCode: 0,
|
||||
@@ -1466,7 +1157,7 @@ class DashboardController {
|
||||
case "scroll-up":
|
||||
return {
|
||||
kind: "text-click",
|
||||
containerName: CONTAINER_NAME,
|
||||
containerName: "",
|
||||
eventType: OsEventTypeList.SCROLL_TOP_EVENT,
|
||||
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
|
||||
systemExitReasonCode: 0,
|
||||
@@ -1476,7 +1167,7 @@ class DashboardController {
|
||||
default:
|
||||
return {
|
||||
kind: "text-click",
|
||||
containerName: CONTAINER_NAME,
|
||||
containerName: "",
|
||||
eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT,
|
||||
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
|
||||
systemExitReasonCode: 0,
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import * as frameTimings from "./frame-timings";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
|
||||
|
||||
export type CommunicatorPhase =
|
||||
| "disconnected"
|
||||
| "connecting"
|
||||
@@ -338,6 +340,19 @@ export class FaceclawCommunicatorBridge {
|
||||
});
|
||||
}
|
||||
|
||||
/** Phone-UI preview of the current composited screen, or null if none yet. */
|
||||
getCompositePreview(): ImageSource | null {
|
||||
if (!global.isAndroid) return null;
|
||||
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA);
|
||||
return bitmap ? new ImageSource(bitmap) : null;
|
||||
}
|
||||
|
||||
/** Save the current composite as a raw 4bpp screenshot; returns the path (empty if none). */
|
||||
saveRawCompositeScreenshot(): string {
|
||||
if (!global.isAndroid) return "";
|
||||
return String(this.communicator.saveRawCompositeScreenshot());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or reconfigure a compositor surface. Geometry changes take effect
|
||||
* when the next frame is submitted.
|
||||
|
||||
@@ -73,6 +73,30 @@ export function listDirectory(path: string): DirectoryEntry[] | null {
|
||||
}
|
||||
}
|
||||
|
||||
/** Write UTF-8 text to a file in the public Downloads directory; returns its path or null. */
|
||||
export function writeTextToDownloads(filename: string, text: string): string | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
const file = new java.io.File(downloads, filename);
|
||||
const bytes = new java.lang.String(text).getBytes("UTF-8");
|
||||
// FileOutputStream rather than Files.write to avoid NativeScript varargs /
|
||||
// overload resolution pitfalls.
|
||||
const stream = new java.io.FileOutputStream(file);
|
||||
try {
|
||||
stream.write(bytes);
|
||||
} finally {
|
||||
stream.close();
|
||||
}
|
||||
return String(file.getAbsolutePath());
|
||||
} catch (error) {
|
||||
console.warn(`writeTextToDownloads failed for ${filename}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Read a UTF-8 text file (size-capped); null when unreadable or too large. */
|
||||
export function readTextFile(path: string): string | null {
|
||||
try {
|
||||
|
||||
@@ -27,6 +27,9 @@ export type PushToTalkOptions = {
|
||||
saveRecording: boolean;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
type CaptureHolder = "ptt" | "continuous";
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
@@ -35,6 +38,11 @@ export class FaceclawVoiceControlBridge {
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private started = false;
|
||||
// The mic is a single shared stream; these are the reasons it is running.
|
||||
// The first holder starts capture (choosing the provider); the mic stops
|
||||
// when the last one releases. Transcript events broadcast to every
|
||||
// listener, so push-to-talk and the Transcribe window both receive text.
|
||||
private readonly captureHolders = new Set<CaptureHolder>();
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: ElevenLabsSttClient | null = null;
|
||||
|
||||
@@ -54,10 +62,34 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Begin push-to-talk capture with the selected provider. */
|
||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("ptt", options);
|
||||
}
|
||||
|
||||
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
|
||||
stopPushToTalk(): void {
|
||||
this.releaseCapture("ptt", true);
|
||||
}
|
||||
|
||||
/** Begin continuous capture (Transcribe): the mic stays on until released. */
|
||||
startContinuousCapture(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("continuous", options);
|
||||
}
|
||||
|
||||
stopContinuousCapture(): void {
|
||||
this.releaseCapture("continuous", false);
|
||||
}
|
||||
|
||||
private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void {
|
||||
if (!global.isAndroid) return;
|
||||
if (this.started) this.stop();
|
||||
const wasIdle = this.captureHolders.size === 0;
|
||||
this.captureHolders.add(holder);
|
||||
if (!wasIdle) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
return;
|
||||
}
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setSaveRecordings(options.saveRecording);
|
||||
@@ -83,21 +115,34 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller?.start("onboard");
|
||||
}
|
||||
|
||||
/** End push-to-talk: stop the mic and, for cloud, commit for a final result. */
|
||||
stopPushToTalk(): void {
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
if (!this.captureHolders.delete(holder)) {
|
||||
// Never held; still finish a dangling cloud commit if one is pending.
|
||||
if (commit) this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
if (this.captureHolders.size > 0) {
|
||||
// Another holder still wants the mic; keep it running.
|
||||
return;
|
||||
}
|
||||
if (!this.started) {
|
||||
this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
// Order matters for cloud: stopping the Java controller flushes any final
|
||||
// decode/PCM; then commit so ElevenLabs finalizes the transcript. The
|
||||
// socket is closed once the committed transcript arrives, or on stop().
|
||||
// decode/PCM; then commit so ElevenLabs finalizes the transcript.
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
this.cloudClient?.finish();
|
||||
if (commit) {
|
||||
this.cloudClient?.finish();
|
||||
} else {
|
||||
this.cloudClient?.stop();
|
||||
this.cloudClient = null;
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.captureHolders.clear();
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
import takeRight from "lodash/takeRight";
|
||||
import { getDefaultSmallFont } from "~/graphics/bdffont";
|
||||
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
|
||||
import { Layer, LayerContext, type DashboardInputEvent } from "../layers";
|
||||
import { GrayImage } from "~/graphics/image";
|
||||
|
||||
export class DebugLogDashboardPlugin extends DashboardPlugin {
|
||||
constructor() {
|
||||
super({
|
||||
id: "input-debug-log",
|
||||
label: "Input debug log",
|
||||
});
|
||||
}
|
||||
|
||||
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
|
||||
const font = getDefaultSmallFont();
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 14, "Input log", 180);
|
||||
const lineHeight = 14;
|
||||
const visibleLineCount = Math.max(1, Math.floor((bounds.height - 28) / lineHeight));
|
||||
const visibleLines = takeRight(state.logLines, visibleLineCount);
|
||||
for (let i = 0; i < visibleLines.length; i++) {
|
||||
image.drawText(font, bounds.x + 8, bounds.y + 26 + i * lineHeight, visibleLines[i]!, 190);
|
||||
}
|
||||
}
|
||||
|
||||
override createFullscreenLayer(getState: () => DashboardPluginState): Layer {
|
||||
return new DebugLogLayer(getState);
|
||||
}
|
||||
}
|
||||
|
||||
class DebugLogLayer implements Layer {
|
||||
private scrollOffset = 0;
|
||||
|
||||
constructor(private readonly getState: () => DashboardPluginState) {}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = new GrayImage(576, 288, 0);
|
||||
image.drawRect(12, 12, 552, 264, 52);
|
||||
image.drawText(font, 22, 16, "Input debug log", 220);
|
||||
|
||||
const logs = this.getState().logLines;
|
||||
const lineHeight = 14;
|
||||
const visibleCount = Math.max(1, Math.floor((288 - 64) / lineHeight));
|
||||
const maxOffset = Math.max(0, logs.length - visibleCount);
|
||||
this.scrollOffset = Math.max(0, Math.min(this.scrollOffset, maxOffset));
|
||||
const start = Math.max(0, maxOffset - this.scrollOffset);
|
||||
const visible = logs.slice(start, start + visibleCount);
|
||||
for (let index = 0; index < visible.length; index++) {
|
||||
image.drawText(font, 22, 42 + index * lineHeight, visible[index]!, 180);
|
||||
}
|
||||
image.drawText(font, 22, 252, "Scroll: browse Double-click: back", 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.scrollOffset += 1;
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.scrollOffset = Math.max(0, this.scrollOffset - 1);
|
||||
return;
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
|
||||
import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import { mediaControllerBridge, type MediaControllerState } from "../../native/media-controller";
|
||||
|
||||
|
||||
export class MusicControllerDashboardPlugin extends DashboardPlugin {
|
||||
constructor() {
|
||||
super({
|
||||
id: "music-controller",
|
||||
label: "Music controller",
|
||||
});
|
||||
}
|
||||
|
||||
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const media = state.media;
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 14, "Now playing", 180);
|
||||
if (!media.accessEnabled) {
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 30, "Enable notification", 150);
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 44, "access for media", 150);
|
||||
return;
|
||||
}
|
||||
if (!media.available) {
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 30, "No active session", 150);
|
||||
return;
|
||||
}
|
||||
|
||||
const titleLines = wrapText(font, media.title || "Unknown title", bounds.width - 20).slice(0, 2);
|
||||
const artistLine = media.artist || media.album || media.packageName || "Unknown source";
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 30, titleLines[0] ?? "", 210);
|
||||
if (titleLines[1]) {
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 44, titleLines[1], 210);
|
||||
}
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 62, artistLine, 160);
|
||||
image.drawText(font, bounds.x + 10, bounds.y + bounds.height - 18, playbackLabel(media), 130);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function playbackLabel(media: MediaControllerState): string {
|
||||
switch (media.playbackState) {
|
||||
case "playing":
|
||||
return "Playing";
|
||||
case "paused":
|
||||
return "Paused";
|
||||
case "buffering":
|
||||
return "Buffering";
|
||||
case "stopped":
|
||||
return "Stopped";
|
||||
case "notification-access-required":
|
||||
return "Access required";
|
||||
default:
|
||||
return media.status || "Idle";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { BdfFont, getDefaultLargeFont, getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { GrayImage, imageFromAsciiArt } from "../../graphics/image";
|
||||
import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin";
|
||||
import { DashboardInputEvent, Layer, LayerContext } from "../layers";
|
||||
import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge";
|
||||
import { isNightscoutSettingsConfigured } from "../dashboard-settings";
|
||||
import { formatAgeShortFromTimestamp, formatTimestamp } from "~/util/date-util";
|
||||
|
||||
const nightscoutLargeFont = getDefaultLargeFont();
|
||||
@@ -10,84 +10,6 @@ const NIGHTSCOUT_STALE_MS = 15 * 60 * 1000;
|
||||
const NIGHTSCOUT_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000;
|
||||
const NIGHTSCOUT_GRAPH_TIME_QUANTUM_MS = 60 * 1000;
|
||||
|
||||
|
||||
export class NightscoutDashboardPlugin extends DashboardPlugin {
|
||||
constructor() {
|
||||
super({
|
||||
id: "nightscout",
|
||||
label: "Nightscout",
|
||||
});
|
||||
}
|
||||
|
||||
override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const nightscout = state.nightscout;
|
||||
const nowMs = Date.now();
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 10, "Nightscout", 180);
|
||||
if (!state.nightscoutConfigured || nightscout.configurationMissing) {
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 30, "Needs setup", 170);
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 44, "Click to configure", 140);
|
||||
return;
|
||||
}
|
||||
if (!nightscout.available || !nightscout.latest) {
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 30, "No glucose data", 150);
|
||||
image.drawText(font, bounds.x + 10, bounds.y + 44, truncateLine(nightscout.status, 26), 120);
|
||||
return;
|
||||
}
|
||||
|
||||
const latest = nightscout.latest;
|
||||
const deltaLabel = formatDelta(nightscout.delta);
|
||||
const glucoseText = `${latest.sgv}`;
|
||||
const glucoseX = bounds.x + bounds.width - 92;
|
||||
image.drawText(nightscoutLargeFont, glucoseX, bounds.y + 10, glucoseText, 230);
|
||||
if (isNightscoutPointStale(latest, nowMs)) {
|
||||
drawNightscoutValueStrikeThrough(image, glucoseX, bounds.y + 22, nightscoutLargeFont.measureText(glucoseText));
|
||||
}
|
||||
drawNightscoutDelta(image, font, bounds.x + bounds.width - 92, bounds.y + 30, deltaLabel, nightscout.direction, 160);
|
||||
image.drawText(
|
||||
font,
|
||||
bounds.x + 10,
|
||||
bounds.y + 30,
|
||||
`IOB ${nightscout.iob === null ? "--" : nightscout.iob.toFixed(1)} COB ${nightscout.cob === null ? "--" : formatWholeNumber(nightscout.cob)}`,
|
||||
160,
|
||||
);
|
||||
image.drawText(
|
||||
font,
|
||||
bounds.x + 10,
|
||||
bounds.y + 44,
|
||||
`Loop ${nightscout.openapsStatusShort} CAGE ${formatAgeShortFromTimestamp(nightscout.cageTimestampMs, nowMs)}`,
|
||||
138,
|
||||
);
|
||||
drawNightscoutGraph(
|
||||
image,
|
||||
{
|
||||
x: bounds.x + 10,
|
||||
y: bounds.y + 60,
|
||||
width: bounds.width - 20,
|
||||
height: Math.max(16, bounds.height - 82),
|
||||
},
|
||||
nightscout,
|
||||
nowMs,
|
||||
font,
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function drawNightscoutDelta(
|
||||
image: GrayImage,
|
||||
font: BdfFont,
|
||||
x: number,
|
||||
y: number,
|
||||
deltaLabel: string,
|
||||
direction: string,
|
||||
shade: number,
|
||||
): void {
|
||||
const text = direction ? `${deltaLabel} ` : deltaLabel;
|
||||
image.drawText(font, x, y, text, shade);
|
||||
drawDirectionIndicator(image, font, x + font.measureText(text), y, direction, shade);
|
||||
}
|
||||
|
||||
function drawDirectionIndicator(
|
||||
image: GrayImage,
|
||||
font: BdfFont,
|
||||
@@ -364,20 +286,18 @@ function drawNightscoutReferenceLine(
|
||||
|
||||
|
||||
export class NightscoutLayer implements Layer {
|
||||
constructor(private readonly getState: () => DashboardPluginState) {}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
// Sized to the hosting stack (the Nightscout app viewport).
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const nightscout = this.getState().nightscout;
|
||||
const nightscout = nightscoutBridge.snapshot();
|
||||
const nowMs = Date.now();
|
||||
const footerY = height - 26;
|
||||
image.drawRect(12, 12, width - 24, height - 24, 52);
|
||||
image.drawText(font, 22, 16, "Nightscout", 220);
|
||||
|
||||
if (!this.getState().nightscoutConfigured || nightscout.configurationMissing) {
|
||||
if (!isNightscoutSettingsConfigured() || nightscout.configurationMissing) {
|
||||
image.drawText(font, 22, 44, "Nightscout needs configuration.", 180);
|
||||
image.drawText(font, 22, 62, "Use Settings > Integrations > Nightscout.", 140);
|
||||
return image;
|
||||
|
||||
+52
-21
@@ -1,50 +1,65 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads on click.
|
||||
*/
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Starting transcription...";
|
||||
private status = "Listening...";
|
||||
// Finalized utterances, plus the live (replace-semantics) partial appended
|
||||
// when painting.
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private actions: LayerActions | null = null;
|
||||
|
||||
start(ctx: LayerContext): void {
|
||||
this.actions = ctx.actions;
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.onTranscript(event);
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
void ctx.actions.setTranscribeRenderActive(true);
|
||||
void Promise.resolve(ctx.actions.startVoiceCapture());
|
||||
}
|
||||
|
||||
paint(): GrayImage {
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const text = this.displayText() || "Listening...";
|
||||
const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64);
|
||||
const wrapped = wrapTranscribeText(font, text, width - 64);
|
||||
|
||||
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
|
||||
image.drawText(font, 24, 24, "Transcribe", 200);
|
||||
image.drawText(font, 24, 46, this.status, 110);
|
||||
image.drawRect(12, 12, width - 24, height - 24, 52);
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const firstLine = Math.max(0, wrapped.length - 10);
|
||||
const bodyTop = 62;
|
||||
const footerY = height - 18;
|
||||
const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / 16));
|
||||
const firstLine = Math.max(0, wrapped.length - bodyLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
const y = 72 + (index - firstLine) * 16;
|
||||
const y = bodyTop + (index - firstLine) * 16;
|
||||
image.drawText(font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(font, 24, 252, "Double-click: back", 110);
|
||||
image.drawText(font, 24, footerY, "Click: save Double-click: back", 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "click") {
|
||||
this.saveTranscript();
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -55,9 +70,17 @@ export class TranscribeLayer implements Layer {
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
void this.actions?.setTranscribeRenderActive(false);
|
||||
void this.actions?.stopVoiceCapture();
|
||||
this.actions = null;
|
||||
}
|
||||
|
||||
private saveTranscript(): void {
|
||||
const text = this.displayText().trim();
|
||||
if (!text) {
|
||||
this.saveNotice = "Nothing to save yet.";
|
||||
return;
|
||||
}
|
||||
const filename = `transcript-${transcriptTimestamp()}.txt`;
|
||||
const path = writeTextToDownloads(filename, `${text}\n`);
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
@@ -80,6 +103,15 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptTimestamp(): string {
|
||||
const now = new Date();
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
return (
|
||||
`${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}` +
|
||||
`-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
@@ -98,4 +130,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri
|
||||
}
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
import { type GrayImage } from "../graphics/image";
|
||||
import { type MediaControllerState } from "../native/media-controller";
|
||||
import { type NightscoutState } from "../native/nightscout-bridge";
|
||||
import { type DashboardPluginId } from "./dashboard-settings";
|
||||
import { type Layer } from "./layers";
|
||||
|
||||
export type DashboardPluginState = {
|
||||
logLines: string[];
|
||||
media: MediaControllerState;
|
||||
nightscout: NightscoutState;
|
||||
nightscoutConfigured: boolean;
|
||||
};
|
||||
|
||||
export type DashboardPluginCardBounds = {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export type DashboardPluginCardRenderArgs = {
|
||||
image: GrayImage;
|
||||
bounds: DashboardPluginCardBounds;
|
||||
selected: boolean;
|
||||
state: DashboardPluginState;
|
||||
};
|
||||
|
||||
export abstract class DashboardPlugin {
|
||||
readonly id: DashboardPluginId;
|
||||
readonly label: string;
|
||||
|
||||
constructor(config: {
|
||||
id: DashboardPluginId;
|
||||
label: string;
|
||||
}) {
|
||||
this.id = config.id;
|
||||
this.label = config.label;
|
||||
}
|
||||
|
||||
abstract renderCard(args: DashboardPluginCardRenderArgs): void;
|
||||
|
||||
createFullscreenLayer(getState: () => DashboardPluginState): Layer | null {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { type DashboardPluginId } from "./dashboard-settings";
|
||||
import { DashboardPlugin } from "./dashboard-plugin";
|
||||
import { NightscoutDashboardPlugin } from "./apps/nightscout";
|
||||
import { MusicControllerDashboardPlugin } from "./apps/music-controller";
|
||||
import { DebugLogDashboardPlugin } from "./apps/debug-log";
|
||||
|
||||
export { DashboardPlugin } from "./dashboard-plugin";
|
||||
export type { DashboardPluginCardBounds, DashboardPluginCardRenderArgs, DashboardPluginState } from "./dashboard-plugin";
|
||||
|
||||
let dashboardPlugins: DashboardPlugin[] | null = null;
|
||||
function getDashboardPlugins(): DashboardPlugin[] {
|
||||
if (!dashboardPlugins) {
|
||||
dashboardPlugins = [
|
||||
{
|
||||
id: "blank",
|
||||
label: "Blank",
|
||||
renderCard: () => {},
|
||||
createFullscreenLayer: () => null,
|
||||
},
|
||||
new DebugLogDashboardPlugin(),
|
||||
new MusicControllerDashboardPlugin(),
|
||||
new NightscoutDashboardPlugin(),
|
||||
];
|
||||
}
|
||||
return dashboardPlugins;
|
||||
};
|
||||
|
||||
export function listDashboardPlugins(): DashboardPlugin[] {
|
||||
return [...getDashboardPlugins()];
|
||||
}
|
||||
|
||||
export function getDashboardPlugin(pluginId: DashboardPluginId): DashboardPlugin {
|
||||
const PLUGIN_MAP = new Map<DashboardPluginId, DashboardPlugin>(
|
||||
getDashboardPlugins().map((plugin) => [plugin.id, plugin]),
|
||||
);
|
||||
return PLUGIN_MAP.get(pluginId) ?? PLUGIN_MAP.get("blank")!;
|
||||
}
|
||||
|
||||
export function isBlankDashboardPlugin(pluginId: DashboardPluginId): boolean {
|
||||
return pluginId === "blank";
|
||||
}
|
||||
@@ -11,24 +11,11 @@ import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from
|
||||
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
|
||||
import { GrayImage } from "~/graphics/image";
|
||||
|
||||
export type DashboardSlotId = "bottom-left" | "bottom-right";
|
||||
export const dashboardSlotIds: DashboardSlotId[] = ["bottom-left", "bottom-right"];
|
||||
export type DashboardPluginId = "blank" | "input-debug-log" | "music-controller" | "nightscout";
|
||||
|
||||
export type DashboardSlotConfig = Record<DashboardSlotId, DashboardPluginId>;
|
||||
export type NightscoutSettings = {
|
||||
siteUrl: string;
|
||||
apiToken: string;
|
||||
};
|
||||
export type BatteryDisplayMode = "icon" | "percentage";
|
||||
export type SystemCardSettings = {
|
||||
showFaceclawLogo: boolean;
|
||||
showBatteryIndicators: boolean;
|
||||
batteryDisplayMode: BatteryDisplayMode;
|
||||
showAndroidNotifications: boolean;
|
||||
showSignalStrength: boolean;
|
||||
};
|
||||
export type SystemCardSettingKey = keyof SystemCardSettings;
|
||||
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "never";
|
||||
|
||||
type ConfigSettingOptions<TValue, TId extends string> = {
|
||||
@@ -171,35 +158,6 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
|
||||
}
|
||||
|
||||
|
||||
const KNOWN_PLUGIN_IDS = new Set<DashboardPluginId>([
|
||||
"blank",
|
||||
"input-debug-log",
|
||||
"music-controller",
|
||||
"nightscout",
|
||||
]);
|
||||
|
||||
|
||||
export const bottomLeftSlotSetting = new ConfigSettingEnum<DashboardPluginId>({
|
||||
id: "bottom-left-slot",
|
||||
label: "Bottom left slot",
|
||||
storageKey: "dashboard.slot.bottomLeft",
|
||||
defaultValue: "blank",
|
||||
values: Array.from(KNOWN_PLUGIN_IDS),
|
||||
});
|
||||
|
||||
export const bottomRightSlotSetting = new ConfigSettingEnum<DashboardPluginId>({
|
||||
id: "bottom-right-slot",
|
||||
label: "Bottom right slot",
|
||||
storageKey: "dashboard.slot.bottomRight",
|
||||
defaultValue: "blank",
|
||||
values: Array.from(KNOWN_PLUGIN_IDS),
|
||||
});
|
||||
|
||||
export const dashboardSlotSettings: Record<DashboardSlotId, ConfigSettingEnum<DashboardPluginId>> = {
|
||||
"bottom-left": bottomLeftSlotSetting,
|
||||
"bottom-right": bottomRightSlotSetting,
|
||||
};
|
||||
|
||||
export const systemCardNameSetting = new ConfigSettingString({
|
||||
id: "dashboard-name",
|
||||
label: "Dashboard name",
|
||||
@@ -211,20 +169,6 @@ export const systemCardNameSetting = new ConfigSettingString({
|
||||
formatValue: (value) => value || "Faceclaw",
|
||||
});
|
||||
|
||||
export const showFaceclawLogoSetting = new ConfigSettingBoolean({
|
||||
id: "showFaceclawLogo",
|
||||
label: "Show Faceclaw Logo",
|
||||
storageKey: "dashboard.systemCard.showFaceclawLogo",
|
||||
defaultValue: true,
|
||||
});
|
||||
|
||||
export const showBatteryIndicatorsSetting = new ConfigSettingBoolean({
|
||||
id: "showBatteryIndicators",
|
||||
label: "Show Battery Indicators",
|
||||
storageKey: "dashboard.systemCard.showBatteryIndicators",
|
||||
defaultValue: true,
|
||||
});
|
||||
|
||||
export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMode>({
|
||||
id: "batteryDisplayMode",
|
||||
label: "Battery display",
|
||||
@@ -234,20 +178,6 @@ export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMod
|
||||
formatValue: batteryDisplayModeLabel,
|
||||
});
|
||||
|
||||
export const showAndroidNotificationsSetting = new ConfigSettingBoolean({
|
||||
id: "showAndroidNotifications",
|
||||
label: "Show Android Notifications",
|
||||
storageKey: "dashboard.systemCard.showAndroidNotifications",
|
||||
defaultValue: true,
|
||||
});
|
||||
|
||||
export const showSignalStrengthSetting = new ConfigSettingBoolean({
|
||||
id: "showSignalStrength",
|
||||
label: "Show Signal Strength",
|
||||
storageKey: "dashboard.systemCard.showSignalStrength",
|
||||
defaultValue: true,
|
||||
});
|
||||
|
||||
export const screenTimeoutSetting = new ConfigSettingEnum<ScreenTimeoutSetting>({
|
||||
id: "screen-timeout",
|
||||
label: "Screen timeout",
|
||||
@@ -388,15 +318,6 @@ export function isNightscoutSettingsConfigured(): boolean {
|
||||
return nightscoutSiteUrlSetting.get().length > 0 && nightscoutApiTokenSetting.get().length > 0;
|
||||
}
|
||||
|
||||
export function normalizePluginId(
|
||||
value: string | null | undefined,
|
||||
fallback: DashboardPluginId = "blank",
|
||||
): DashboardPluginId {
|
||||
if (value && KNOWN_PLUGIN_IDS.has(value as DashboardPluginId)) {
|
||||
return value as DashboardPluginId;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function normalizeSystemCardName(name: string | null | undefined): string {
|
||||
const normalized = (name ?? "").replace(/[\x00-\x1f]+/g, " ").replace(/\s+/g, " ").trim();
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { nightscoutBridge } from "../native/nightscout-bridge";
|
||||
import { getDashboardPlugin, type DashboardPluginCardBounds, type DashboardPluginState } from "./dashboard-plugins";
|
||||
import { EditTextSettingLayer, isNightscoutSettingsConfigured, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings";
|
||||
import { Layer, LayerActions, LayerStack, type DashboardInputEvent } from "./layers";
|
||||
import { drawSystemCard } from "./dashboard/system-card";
|
||||
import { createRootMenuLayer } from "./dashboard/root-menu";
|
||||
import { inputEventToString, shell } from "./shell/shell";
|
||||
|
||||
type DashboardCardId = "system" | DashboardSlotId;
|
||||
export type DashboardBatteryLevels = {
|
||||
headset: number | null;
|
||||
headsetCharging: boolean | null;
|
||||
ring: number | null;
|
||||
};
|
||||
|
||||
type DashboardState = {
|
||||
logLines: string[];
|
||||
battery: DashboardBatteryLevels;
|
||||
};
|
||||
|
||||
export let dashboardState: DashboardState = {
|
||||
logLines: [] as string[],
|
||||
battery: {
|
||||
headset: null,
|
||||
headsetCharging: null,
|
||||
ring: null,
|
||||
} as DashboardBatteryLevels,
|
||||
};
|
||||
|
||||
const dashboardActions: LayerActions = {
|
||||
requestRender: () => {},
|
||||
disconnect: () => {},
|
||||
startTextSettingEdit: () => {},
|
||||
endTextSettingEdit: () => {},
|
||||
setTranscribeRenderActive: () => {},
|
||||
startVoiceCapture: () => {},
|
||||
stopVoiceCapture: () => {},
|
||||
playBuzzerSequence: () => {},
|
||||
};
|
||||
export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 };
|
||||
|
||||
/**
|
||||
* Input handler for the dashboard window. Global gestures (wake, sleep,
|
||||
* long-press/push-to-talk) are consumed by the shell before this is called.
|
||||
*/
|
||||
export async function receiveDashboardWindowInput(inputEvent: DashboardInputEvent): Promise<void> {
|
||||
dashboardState.logLines.push(inputEventToString(inputEvent));
|
||||
await dashboardLayers.handleInput(inputEvent);
|
||||
}
|
||||
|
||||
export function logToDashboard(message: string): void {
|
||||
dashboardState.logLines.push(message);
|
||||
}
|
||||
|
||||
export function setDashboardActions(actions: Partial<LayerActions>): void {
|
||||
dashboardLayers.setActions(actions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the glasses-side text-setting edit page, e.g. when the user finishes
|
||||
* typing on the phone. Returns whether the display changed.
|
||||
*/
|
||||
export function closeDashboardTextSettingEditor(): boolean {
|
||||
return dashboardLayers.popIfTop((layer) => layer instanceof EditTextSettingLayer);
|
||||
}
|
||||
|
||||
export function drawDashboard(): GrayImage {
|
||||
if (!shell.isScreenOn()) {
|
||||
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
}
|
||||
return dashboardLayers.paint();
|
||||
}
|
||||
|
||||
export function resetDashboardSleepTimerAndWake(nowMs = Date.now()): boolean {
|
||||
if (!shell.wake("sidebar", nowMs)) return false;
|
||||
dashboardLayers.clearToBase();
|
||||
dashboardLayers.push(createRootMenuLayer());
|
||||
return true;
|
||||
}
|
||||
|
||||
export function setDashboardBatteryLevels(levels: Partial<DashboardBatteryLevels>): void {
|
||||
dashboardState.battery = {
|
||||
...dashboardState.battery,
|
||||
...levels,
|
||||
};
|
||||
}
|
||||
|
||||
export function getPluginIdForSlot(slot: DashboardSlotId): DashboardPluginId {
|
||||
switch (slot) {
|
||||
case "bottom-left":
|
||||
return bottomLeftSlotSetting.get();
|
||||
case "bottom-right":
|
||||
return bottomRightSlotSetting.get();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DashboardLayer implements Layer {
|
||||
paint(): GrayImage {
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const pluginState = getPluginState();
|
||||
drawSystemCard(image, getCardBounds("system"));
|
||||
|
||||
for (const slot of dashboardSlotIds) {
|
||||
const bounds = getCardBounds(slot);
|
||||
const pluginId = getPluginIdForSlot(slot);
|
||||
getDashboardPlugin(pluginId).renderCard({
|
||||
image,
|
||||
bounds,
|
||||
selected: false,
|
||||
state: pluginState,
|
||||
});
|
||||
}
|
||||
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: { stack: LayerStack }): void {
|
||||
if (event.type === "click") {
|
||||
ctx.stack.push(createRootMenuLayer());
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "double-click") {
|
||||
// Backing out of the window's root returns focus to the shell sidebar.
|
||||
shell.yieldFocusToSidebar();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function getPluginState(): DashboardPluginState {
|
||||
return {
|
||||
logLines: dashboardState.logLines,
|
||||
media: mediaControllerBridge.snapshot(),
|
||||
nightscout: nightscoutBridge.snapshot(),
|
||||
nightscoutConfigured: isNightscoutSettingsConfigured(),
|
||||
};
|
||||
}
|
||||
|
||||
function getCardBounds(card: DashboardCardId): DashboardPluginCardBounds {
|
||||
switch (card) {
|
||||
case "system":
|
||||
return {
|
||||
x: G2_LENS_WIDTH / 2 + 2,
|
||||
y: 0,
|
||||
width: G2_LENS_WIDTH / 2 - 2,
|
||||
height: G2_LENS_HEIGHT / 2 - 2,
|
||||
};
|
||||
case "bottom-left":
|
||||
return {
|
||||
x: 0,
|
||||
y: G2_LENS_HEIGHT / 2 + 2,
|
||||
width: G2_LENS_WIDTH / 2 - 2,
|
||||
height: G2_LENS_HEIGHT / 2 - 2,
|
||||
};
|
||||
case "bottom-right":
|
||||
return {
|
||||
x: G2_LENS_WIDTH / 2 + 2,
|
||||
y: G2_LENS_HEIGHT / 2 + 2,
|
||||
width: G2_LENS_WIDTH / 2 - 2,
|
||||
height: G2_LENS_HEIGHT / 2 - 2,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const dashboardLayers = new LayerStack(new DashboardLayer(), dashboardActions);
|
||||
dashboardLayers.push(createRootMenuLayer());
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
import { getDefaultSmallFont } from "~/graphics/bdffont";
|
||||
import { getPluginIdForSlot, getPluginState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
|
||||
import { getDashboardPlugin, isBlankDashboardPlugin } from "../dashboard-plugins";
|
||||
import { DashboardSlotId, dashboardSlotIds, isNightscoutSettingsConfigured } from "../dashboard-settings";
|
||||
import { DashboardInputEvent, LayerContext } from "../layers";
|
||||
import { MenuLayer } from "../menu";
|
||||
import { createNightscoutSettingsMenuLayer } from "./settings-menus";
|
||||
import { TranscribeLayer } from "../apps/transcribe";
|
||||
import { GrayImage } from "~/graphics/image";
|
||||
import { shell } from "../shell/shell";
|
||||
|
||||
const rootMenuFont = getDefaultSmallFont();
|
||||
|
||||
class RootMenuLayer extends MenuLayer {
|
||||
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
|
||||
if (event.type === "double-click") {
|
||||
// Backing out of the window's root returns focus to the shell sidebar.
|
||||
shell.yieldFocusToSidebar();
|
||||
return;
|
||||
}
|
||||
await super.handleInput(event, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
export function createRootMenuLayer(): MenuLayer {
|
||||
return new RootMenuLayer(
|
||||
null,
|
||||
[
|
||||
{
|
||||
label: "Apps",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(createAppsMenuLayer());
|
||||
},
|
||||
},
|
||||
...dashboardSlotIds
|
||||
.filter((slot) => !isBlankDashboardPlugin(getPluginIdForSlot(slot)))
|
||||
.map((slot) => ({
|
||||
label: pluginLabelForSlot(slot),
|
||||
onSelect: (ctx: LayerContext) => {
|
||||
openDashboardSlot(slot, ctx);
|
||||
},
|
||||
render: ({ image, x, y }: { image: GrayImage; x: number; y: number }) => {
|
||||
image.drawText(rootMenuFont, x, y + 3, pluginLabelForSlot(slot), 200);
|
||||
},
|
||||
})),
|
||||
],
|
||||
TOP_LEFT_MENU_LAYOUT,
|
||||
);
|
||||
}
|
||||
|
||||
function createAppsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Apps",
|
||||
[
|
||||
{
|
||||
label: "Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
const layer = new TranscribeLayer();
|
||||
ctx.stack.push(layer);
|
||||
layer.start(ctx);
|
||||
},
|
||||
},
|
||||
],
|
||||
TOP_LEFT_MENU_LAYOUT,
|
||||
);
|
||||
}
|
||||
|
||||
function openDashboardSlot(slot: DashboardSlotId, ctx: LayerContext): void {
|
||||
const pluginId = getPluginIdForSlot(slot);
|
||||
if (isBlankDashboardPlugin(pluginId)) {
|
||||
return;
|
||||
}
|
||||
if (pluginId === "nightscout" && !isNightscoutSettingsConfigured()) {
|
||||
ctx.stack.push(createNightscoutSettingsMenuLayer());
|
||||
return;
|
||||
}
|
||||
const layer = getDashboardPlugin(pluginId).createFullscreenLayer?.(getPluginState);
|
||||
if (layer) {
|
||||
ctx.stack.push(layer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function pluginLabelForSlot(slot: DashboardSlotId): string {
|
||||
return getDashboardPlugin(getPluginIdForSlot(slot)).label;
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
import { APP_VIEWPORT } from "../shell/geometry";
|
||||
import { type MenuLayout } from "../menu";
|
||||
|
||||
// Settings menus render in both the dashboard window (full lens) and the
|
||||
// Settings app window (viewport-sized); cap heights to the smaller of the two.
|
||||
const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 };
|
||||
import { MenuLayer, type MenuLayout } from "../menu";
|
||||
import { shell } from "../shell/shell";
|
||||
import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
|
||||
import { MenuLayer } from "../menu";
|
||||
import { batteryDisplayModeSetting, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
|
||||
import { AboutPage } from "./about-page";
|
||||
|
||||
// Rendered in the Settings app window (viewport-sized); cap height to it.
|
||||
const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 };
|
||||
|
||||
export function createSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Settings",
|
||||
@@ -19,12 +17,6 @@ export function createSettingsMenuLayer(): MenuLayer {
|
||||
ctx.stack.push(createDisplaySettingsMenuLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Dashboard",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(createDashboardSettingsMenuLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Voice",
|
||||
onSelect: (ctx) => {
|
||||
@@ -96,10 +88,12 @@ function createDisplaySettingsMenuLayer(): MenuLayer {
|
||||
"Settings > Display",
|
||||
[
|
||||
enumSettingMenuItem(screenTimeoutSetting, {
|
||||
onChange: (ctx, newValue, oldValue) => {
|
||||
onChange: () => {
|
||||
shell.noteUserActivity();
|
||||
},
|
||||
}),
|
||||
// Controls the top-bar battery indicators (icon vs percentage).
|
||||
enumSettingMenuItem(batteryDisplayModeSetting),
|
||||
],
|
||||
SETTINGS_MENU_LAYOUT,
|
||||
);
|
||||
@@ -116,39 +110,6 @@ function createVoiceSettingsMenuLayer(): MenuLayer {
|
||||
);
|
||||
}
|
||||
|
||||
function createDashboardSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Settings > Dashboard",
|
||||
[
|
||||
{
|
||||
label: "System Card",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(createSystemCardSettingsMenuLayer());
|
||||
},
|
||||
},
|
||||
...dashboardSlotIds.map((slot) => enumSettingMenuItem(dashboardSlotSettings[slot], {
|
||||
style: "submenu",
|
||||
})),
|
||||
],
|
||||
SETTINGS_MENU_LAYOUT,
|
||||
);
|
||||
}
|
||||
|
||||
function createSystemCardSettingsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Settings > Dashboard > System Card",
|
||||
[
|
||||
textSettingMenuItem(systemCardNameSetting),
|
||||
toggleSettingMenuItem(showFaceclawLogoSetting),
|
||||
toggleSettingMenuItem(showBatteryIndicatorsSetting),
|
||||
enumSettingMenuItem(batteryDisplayModeSetting),
|
||||
toggleSettingMenuItem(showAndroidNotificationsSetting),
|
||||
toggleSettingMenuItem(showSignalStrengthSetting),
|
||||
],
|
||||
SETTINGS_MENU_LAYOUT,
|
||||
);
|
||||
}
|
||||
|
||||
function createIntegrationsMenuLayer(): MenuLayer {
|
||||
return new MenuLayer(
|
||||
"Settings > Integrations",
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { spanCurrent } from "../../native/frame-timings";
|
||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
|
||||
import { BATTERY_ICON_WIDTH, drawBattery } from "../../graphics/battery";
|
||||
import { readActiveNotificationIcons } from "../../native/notification-icons";
|
||||
import { readPhoneBatteryState } from "../../native/phone-battery";
|
||||
import { readSystemStatusIcons } from "../../native/system-status-icons";
|
||||
import { type DashboardPluginCardBounds } from "../dashboard-plugins";
|
||||
import { dashboardState } from "../dashboard";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "~/graphics/bdffont";
|
||||
import { getDashboardLogo } from "~/graphics/logo";
|
||||
import { batteryDisplayModeSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, showSignalStrengthSetting, systemCardNameSetting } from "../dashboard-settings";
|
||||
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const SYSTEM_CARD_ITEM_HEIGHT = 38;
|
||||
const SYSTEM_CARD_ITEM_GAP = 2;
|
||||
const BATTERY_ITEM_Y_OFFSET = 4;
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
|
||||
type SystemCardFlowItem =
|
||||
| { type: "battery"; label: string; percentCharge: number; isCharging: boolean }
|
||||
| { type: "notification"; icon: GrayImage };
|
||||
|
||||
const smallFont = getDefaultSmallFont();
|
||||
const mediumFont = getDefaultMediumFont();
|
||||
|
||||
function drawSystemCardFlowItems(image: GrayImage, bounds: DashboardPluginCardBounds): void {
|
||||
const left = bounds.x + 10;
|
||||
const top = bounds.y + 84;
|
||||
const right = bounds.x + bounds.width - 10;
|
||||
const bottom = bounds.y + bounds.height - 6;
|
||||
const items: SystemCardFlowItem[] = [];
|
||||
if (showAndroidNotificationsSetting.get()) {
|
||||
const maxNotificationIcons = Math.max(0, ((right - left) / Math.max(1, NOTIFICATION_ICON_SIZE + SYSTEM_CARD_ITEM_GAP)) | 0) * 2;
|
||||
const { icons, stale } = readActiveNotificationIcons(maxNotificationIcons, renderPassAllowsStaleData());
|
||||
if (stale) {
|
||||
noteStaleDataUsed();
|
||||
}
|
||||
for (const icon of icons) {
|
||||
items.push({ type: "notification", icon });
|
||||
}
|
||||
}
|
||||
items.push(...collectBatteryItems());
|
||||
if (showSignalStrengthSetting.get()) {
|
||||
for (const icon of readSystemStatusIcons()) {
|
||||
items.push({ type: "notification", icon });
|
||||
}
|
||||
}
|
||||
|
||||
let itemX = left;
|
||||
let itemY = top;
|
||||
for (const item of items) {
|
||||
const itemWidth = systemCardFlowItemWidth(item);
|
||||
if (itemX > left && itemX + itemWidth > right) {
|
||||
itemX = left;
|
||||
itemY += SYSTEM_CARD_ITEM_HEIGHT + SYSTEM_CARD_ITEM_GAP;
|
||||
}
|
||||
if (itemY + SYSTEM_CARD_ITEM_HEIGHT > bottom) {
|
||||
break;
|
||||
}
|
||||
if (item.type === "battery") {
|
||||
drawBatteryFlowItem(image, itemX, itemY, item);
|
||||
} else {
|
||||
image.bitBlt(
|
||||
item.icon,
|
||||
itemX,
|
||||
itemY + ((SYSTEM_CARD_ITEM_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0,
|
||||
);
|
||||
}
|
||||
itemX += itemWidth + SYSTEM_CARD_ITEM_GAP;
|
||||
}
|
||||
}
|
||||
|
||||
function systemCardFlowItemWidth(item: SystemCardFlowItem): number {
|
||||
if (item.type === "notification") {
|
||||
return NOTIFICATION_ICON_SIZE;
|
||||
}
|
||||
if (batteryDisplayModeSetting.get() === "percentage") {
|
||||
const percentText = formatBatteryPercent(item.percentCharge);
|
||||
return Math.max(smallFont.measureText(percentText), smallFont.measureText(item.label));
|
||||
}
|
||||
return Math.max(BATTERY_ICON_WIDTH, smallFont.measureText(item.label));
|
||||
}
|
||||
|
||||
function collectBatteryItems(): SystemCardFlowItem[] {
|
||||
if (!showBatteryIndicatorsSetting.get()) return [];
|
||||
const phone = readPhoneBatteryState();
|
||||
const items: SystemCardFlowItem[] = [];
|
||||
addBatteryItem(items, "Phone", phone.battery, phone.charging);
|
||||
addBatteryItem(items, "G2", dashboardState.battery.headset, dashboardState.battery.headsetCharging);
|
||||
addBatteryItem(items, "R1", dashboardState.battery.ring, null);
|
||||
return items;
|
||||
}
|
||||
|
||||
function addBatteryItem(
|
||||
items: SystemCardFlowItem[],
|
||||
label: string,
|
||||
percentCharge: number | null,
|
||||
isCharging: boolean | null,
|
||||
): void {
|
||||
if (percentCharge === null || !Number.isFinite(percentCharge)) return;
|
||||
items.push({
|
||||
type: "battery",
|
||||
label,
|
||||
percentCharge,
|
||||
isCharging: Boolean(isCharging),
|
||||
});
|
||||
}
|
||||
|
||||
function formatBatteryPercent(percentCharge: number): string {
|
||||
const clamped = Math.max(0, Math.min(100, Math.round(percentCharge)));
|
||||
return `${clamped}%`;
|
||||
}
|
||||
|
||||
function drawBatteryFlowItem(image: GrayImage, x: number, y: number, item: Extract<SystemCardFlowItem, { type: "battery" }>): void {
|
||||
const itemWidth = systemCardFlowItemWidth(item);
|
||||
const labelX = x + Math.max(0, ((itemWidth - smallFont.measureText(item.label)) / 2) | 0);
|
||||
image.drawText(smallFont, labelX, y + BATTERY_ITEM_Y_OFFSET, item.label, 150);
|
||||
if (batteryDisplayModeSetting.get() === "percentage") {
|
||||
drawBatteryPercentFlowItem(image, x, y, item, itemWidth);
|
||||
return;
|
||||
}
|
||||
const battery = drawBattery(item.percentCharge, item.isCharging);
|
||||
image.bitBlt(battery, x + ((itemWidth - battery.width) / 2) | 0, y + 16 + BATTERY_ITEM_Y_OFFSET);
|
||||
}
|
||||
|
||||
function drawBatteryPercentFlowItem(
|
||||
image: GrayImage,
|
||||
x: number,
|
||||
y: number,
|
||||
item: Extract<SystemCardFlowItem, { type: "battery" }>,
|
||||
itemWidth: number,
|
||||
): void {
|
||||
const percentText = formatBatteryPercent(item.percentCharge);
|
||||
const textWidth = smallFont.measureText(percentText);
|
||||
const textX = x + Math.max(0, ((itemWidth - textWidth) / 2) | 0);
|
||||
const textY = y + 16 + BATTERY_ITEM_Y_OFFSET;
|
||||
if (item.isCharging) {
|
||||
const padX = 2;
|
||||
const padY = 1;
|
||||
image.fillRect(textX - padX, textY - padY, textWidth + padX * 2, smallFont.lineHeight + padY * 2, 255);
|
||||
image.drawText(smallFont, textX, textY, percentText, 0);
|
||||
return;
|
||||
}
|
||||
image.drawText(smallFont, textX, textY, percentText, 200);
|
||||
}
|
||||
|
||||
function formatDashboardDate(now: Date): string {
|
||||
return `${WEEKDAYS[now.getDay()]} ${MONTHS[now.getMonth()]} ${now.getDate()} ${pad2(now.getHours())}:${pad2(now.getMinutes())}`;
|
||||
}
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
export function drawSystemCard(image: GrayImage, bounds: DashboardPluginCardBounds): void {
|
||||
spanCurrent("draw-system-card", () => {
|
||||
const now = new Date();
|
||||
const logo = getDashboardLogo();
|
||||
|
||||
if (logo && showFaceclawLogoSetting.get()) {
|
||||
image.bitBlt(logo, bounds.x + 10, bounds.y + 10);
|
||||
}
|
||||
const infoX = logo && showFaceclawLogoSetting.get() ? bounds.x + 92 : bounds.x + 10;
|
||||
image.drawText(mediumFont, infoX, bounds.y + 10, systemCardNameSetting.get(), 200);
|
||||
image.drawText(mediumFont, infoX, bounds.y + 32, formatDashboardDate(now), 200);
|
||||
drawSystemCardFlowItems(image, bounds);
|
||||
});
|
||||
}
|
||||
|
||||
+5
-3
@@ -18,11 +18,13 @@ export type LayerActions = {
|
||||
disconnect: () => Promise<void> | void;
|
||||
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
|
||||
endTextSettingEdit: () => Promise<void> | void;
|
||||
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
|
||||
/** Start voice capture with the provider chosen in settings. */
|
||||
/** Start push-to-talk voice capture with the provider chosen in settings. */
|
||||
startVoiceCapture: () => Promise<void> | void;
|
||||
/** Stop voice capture; for a cloud provider this also commits for a final result. */
|
||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
startContinuousVoiceCapture: () => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: () => Promise<void> | void;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge";
|
||||
import { getPluginState } from "../dashboard";
|
||||
import { NightscoutLayer } from "../apps/nightscout";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
@@ -35,7 +34,7 @@ export function createNightscoutAppWindow(options: InProcessAppOptions): InProce
|
||||
iconLetter: "Ns",
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(new NightscoutLayer(getPluginState)),
|
||||
baseLayer: new YieldAtRootLayer(new NightscoutLayer()),
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
|
||||
@@ -66,9 +66,10 @@ const noopActions: LayerActions = {
|
||||
disconnect: () => {},
|
||||
startTextSettingEdit: () => {},
|
||||
endTextSettingEdit: () => {},
|
||||
setTranscribeRenderActive: () => {},
|
||||
startVoiceCapture: () => {},
|
||||
stopVoiceCapture: () => {},
|
||||
startContinuousVoiceCapture: () => {},
|
||||
stopContinuousVoiceCapture: () => {},
|
||||
playBuzzerSequence: () => {},
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { imageFromAsciiArt } from "../../graphics/image";
|
||||
import { TranscribeLayer } from "../apps/transcribe";
|
||||
import type { LayerActions } from "../layers";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
type InProcessAppOptions,
|
||||
type InProcessWindow,
|
||||
} from "./in-process-window";
|
||||
import { shell } from "./shell";
|
||||
|
||||
export const TRANSCRIBE_WINDOW_ID = "transcribe";
|
||||
export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
|
||||
|
||||
const TRAY_ICON_ID = "transcribe";
|
||||
|
||||
// 16x18 microphone glyph for the top-bar tray.
|
||||
const MIC_ICON = imageFromAsciiArt(
|
||||
[
|
||||
" #### ",
|
||||
" ###### ",
|
||||
" # # ",
|
||||
" # # ",
|
||||
" # # ",
|
||||
" # # ",
|
||||
" # # ",
|
||||
" # # # # ",
|
||||
" # # # # ",
|
||||
" # #### # ",
|
||||
" # # ",
|
||||
" ###### ",
|
||||
" # ",
|
||||
" # ",
|
||||
" ###### ",
|
||||
],
|
||||
220,
|
||||
);
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: () => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The Transcribe app: live speech-to-text in its own window. While open it
|
||||
* holds continuous mic capture (so push-to-talk can share the stream) and
|
||||
* shows a microphone tray icon in the top bar.
|
||||
*/
|
||||
export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow {
|
||||
const layer = new TranscribeLayer();
|
||||
const actions: LayerActions = { ...options.actions };
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
title: "Transcribe",
|
||||
iconLetter: "Tr",
|
||||
closeable: true,
|
||||
actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
options.stopContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// start() wires the layer's transcript/status subscriptions; the LayerStack
|
||||
// exposes its context through a paint, but the layer needs it up front, so
|
||||
// build a minimal context here.
|
||||
layer.start({ stack: app.stack, actions });
|
||||
options.startContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
}
|
||||
Reference in new issue
Block a user