diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 5460bfc4..cfa11f51 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -268,6 +268,31 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { compositor.configureScreen(width, height); } + /** + * The current composited screen as a phone-UI preview bitmap, or null + * before any surface has been configured. Built from the compositor so + * the preview reflects every surface (chrome + whichever app is + * foreground), including worker-app frames the TS side never sees. + */ + public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return null; + } + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma); + } + + /** Save the current composite as a raw 4bpp screenshot; returns the path or "". */ + public String saveRawCompositeScreenshot() throws java.io.IOException { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return ""; + } + return ScreenshotUtil.saveRawScreenshot( + appContext, java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height); + } + /** Show or hide a compositor surface; takes effect at the next composite. */ public void setSurfaceVisible(String id, boolean visible) { compositor.setSurfaceVisible(id, visible); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java index a02baa3d..64249af9 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java @@ -217,6 +217,38 @@ public final class SurfaceCompositor { } } + /** + * Current composited pixels for the phone-side preview / screenshot, or + * null before the screen is configured. Does not consume a sequence + * number (it is never stored as the desired frame). + */ + public Composite previewComposite() { + synchronized (lock) { + if (screenWidth <= 0 || screenHeight <= 0) { + return null; + } + byte[] gray = buildGrayLocked(); + return new Composite(gray, screenWidth, screenHeight, "preview", 0); + } + } + + private byte[] buildGrayLocked() { + byte[] gray = new byte[screenWidth * screenHeight]; + if (blanked) { + return gray; + } + List ordered = new ArrayList<>(surfaces.values()); + ordered.sort(Comparator + .comparingInt((Surface s) -> s.zOrder) + .thenComparing(s -> s.id)); + for (Surface surface : ordered) { + if (surface.visible) { + blendLocked(gray, surface); + } + } + return gray; + } + private Composite compositeLocked() { byte[] gray = new byte[screenWidth * screenHeight]; if (blanked) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 4a777beb..5942d72a 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -14,17 +14,7 @@ import { firmwareIncompatibilityMessage } from "./firmware-compat"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; -import { - closeDashboardTextSettingEditor, - drawDashboard, - logToDashboard, - receiveDashboardWindowInput, - resetDashboardSleepTimerAndWake, - setDashboardBatteryLevels, - setDashboardActions, -} from "../ui/dashboard"; -import { inputEventToString, rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; -import { makeLetterWindowIcon } from "../ui/shell/chrome-layer"; +import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; import { WorkerAppHost } from "../ui/shell/worker-window"; import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app"; import { @@ -55,6 +45,11 @@ import { NIGHTSCOUT_WINDOW_ID, } from "../ui/shell/nightscout-app"; import { createMusicAppWindow, MUSIC_SURFACE_ID, MUSIC_WINDOW_ID } from "../ui/shell/music-app"; +import { + createTranscribeAppWindow, + TRANSCRIBE_SURFACE_ID, + TRANSCRIBE_WINDOW_ID, +} from "../ui/shell/transcribe-app"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { APP_VIEWPORT } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; @@ -72,7 +67,6 @@ import { voiceProviderSetting, type ConfigSettingString, } from "../ui/dashboard-settings"; -import { saveRawScreenshot } from "../native/raw-screenshot"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting"; @@ -94,17 +88,15 @@ export type DashboardSnapshot = { }; type DashboardListener = (snapshot: DashboardSnapshot) => void; -type LogLevel = "debug"|"info"|"warn"|"error"; -const CONTAINER_NAME = "dashboard"; -// Compositor surfaces: the dashboard window in the app viewport, and the -// shell chrome (sidebar + top bar + overlays) above it with color-key -// transparency. -const DASHBOARD_WINDOW_SURFACE_ID = "window:dashboard"; +// The shell chrome (sidebar + top bar + overlays) composites above all app +// window surfaces with color-key transparency. const SHELL_SURFACE_ID = "shell"; -const DASHBOARD_INTERVAL_MS = 60_000; +// Top-bar clock refresh; the phone-side preview polls the Java composite so +// it reflects every app (including worker apps the TS side never renders). +const SHELL_REFRESH_INTERVAL_MS = 60_000; +const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; -const TRANSCRIBE_RENDER_INTERVAL_MS = 250; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000; const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000; @@ -119,32 +111,6 @@ function formatTimestamp(date: Date): string { return date.toISOString().slice(11, 23); } -function pad2(value: number): string { - return String(value).padStart(2, "0"); -} - -function formatTime24h(date: Date, includeSeconds: boolean): string { - const hours = date.getHours(); - const minutes = pad2(date.getMinutes()); - const seconds = pad2(date.getSeconds()); - return includeSeconds - ? `${hours}:${minutes}:${seconds}` - : `${hours}:${minutes}`; -} - -function formatUtcOffset(date: Date): string { - const offsetMinutes = -date.getTimezoneOffset(); - const sign = offsetMinutes >= 0 ? "+" : "-"; - const absoluteMinutes = Math.abs(offsetMinutes); - const hours = Math.floor(absoluteMinutes / 60); - const minutes = absoluteMinutes % 60; - return `UTC${sign}${pad2(hours)}:${pad2(minutes)}`; -} - -function formatForStatus(date: Date): string { - return formatTime24h(date, true); -} - function eventName(eventType: number): string { return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`; } @@ -153,25 +119,6 @@ function sourceName(eventSource: number): string { return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`; } -function normalizeContainerName(name: string): string { - return name.replace(/[^\x20-\x7e]+/g, ""); -} - -function isDashboardContainerName(name: string): boolean { - const normalized = normalizeContainerName(name); - return normalized === CONTAINER_NAME || normalized.includes(CONTAINER_NAME); -} - -/** Top-left viewport-sized crop of a full-lens dashboard paint. */ -function cropToViewport(image: GrayImage): GrayImage { - if (image.width === APP_VIEWPORT.width && image.height === APP_VIEWPORT.height) { - return image; - } - const cropped = new GrayImage(APP_VIEWPORT.width, APP_VIEWPORT.height, 0); - cropped.bitBlt(image, 0, 0, { width: APP_VIEWPORT.width, height: APP_VIEWPORT.height }); - return cropped; -} - class DashboardController { private phase: ConnectionPhase = "disconnected"; private status = "Disconnected."; @@ -185,9 +132,9 @@ class DashboardController { private readonly listeners = new Set(); private communicator: FaceclawCommunicatorBridge | null = null; - private dashboardTimer: ReturnType | null = null; + private shellRefreshTimer: ReturnType | null = null; + private previewTimer: ReturnType | null = null; private screenTimeoutTimer: ReturnType | null = null; - private transcribeRenderTimer: ReturnType | null = null; private offState: (() => void) | null = null; private offLog: (() => void) | null = null; private offRing: (() => void) | null = null; @@ -195,23 +142,14 @@ class DashboardController { private offEvenAppConflict: (() => void) | null = null; private offFrameMetrics: (() => void) | null = null; private offFirmwareInfo: (() => void) | null = null; - private offMedia: (() => void) | null = null; - private offNightscout: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; private offVoiceWakeWord: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; private lastInput = "waiting..."; private lastSys = "none yet"; - private renderInProgress = false; - private renderQueued = false; - private queuedRenderReason: "initial" | "interval" = "interval"; - private queuedFrameId = 0; private shellRenderInProgress = false; private shellRenderQueued = false; private nextShellRenderWantsFreshData = false; - // Last painted surface images, for compositing the phone-side preview. - private lastShellImage: GrayImage | null = null; - private lastWindowImage: GrayImage | null = null; // One shared worker per app hosts all its windows; spawned on first launch. private readonly appHosts = new Map(); private nextWindowSerial = 1; @@ -223,29 +161,19 @@ class DashboardController { private sharedActions!: Omit; private lastForegroundNotificationUpdateAtMs = 0; private lastConnectedPreviewUpdateAtMs = 0; - // Consumed by the next renderDashboard: repaint with data sources not - // allowed to serve stale caches (set after a frame painted with stale data). - private nextRenderWantsFreshData = false; constructor() { const sharedActions = { disconnect: () => this.disconnect(), startTextSettingEdit: (setting: ConfigSettingString) => this.startTextSettingEdit(setting), endTextSettingEdit: () => this.endTextSettingEdit(), - setTranscribeRenderActive: (active: boolean) => this.setTranscribeRenderActive(active), startVoiceCapture: () => this.startVoiceCapture(), stopVoiceCapture: () => this.stopVoiceCapture(), + startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), + stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; - setDashboardActions({ - ...sharedActions, - requestRender: () => { - void this.requestRender("interval").catch((error) => { - this.appendLog(`requested render failed: ${this.formatError(error)}`); - }); - }, - }); shell.configure({ actions: { ...sharedActions, @@ -282,6 +210,7 @@ class DashboardController { { appId: "teleprompt", label: "Teleprompt" }, { appId: "music", label: "Music" }, { appId: "nightscout", label: "Nightscout" }, + { appId: "transcribe", label: "Transcribe" }, { appId: "notifications", label: "Notifications" }, { appId: "debug-tests", label: "Debug tests" }, { appId: "settings", label: "Settings" }, @@ -294,26 +223,6 @@ class DashboardController { }, }), ); - shell.registerWindow({ - appId: "dashboard", - windowId: "dashboard", - title: "Dashboard", - surfaceId: DASHBOARD_WINDOW_SURFACE_ID, - closeable: false, - drawIcon: makeLetterWindowIcon("D"), - handleInput: async (event, frameId) => { - await receiveDashboardWindowInput(event); - await this.requestRender("interval", frameId); - }, - requestRender: () => { - void this.requestRender("interval").catch((error) => { - this.appendLog(`window render failed: ${this.formatError(error)}`); - }); - }, - setForeground: (foreground) => { - this.setWindowSurfaceVisible(DASHBOARD_WINDOW_SURFACE_ID, foreground); - }, - }); this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => { void this.handleAndroidNotificationPosted(notificationKey).catch((error) => { this.appendLog(`notification wake failed: ${this.formatError(error)}`); @@ -336,12 +245,12 @@ class DashboardController { } /** - * Save the current dashboard as the raw headerless 4bpp frame buffer the - * wire compressor sees; test data for compression experiments. + * Save the current composited screen as the raw headerless 4bpp frame + * buffer the wire compressor sees; test data for compression experiments. */ saveRawDashboardScreenshot(): string { - const path = saveRawScreenshot(drawDashboard()); - this.appendLog(`raw screenshot saved: ${path}`); + const path = this.communicator?.saveRawCompositeScreenshot() ?? ""; + this.appendLog(path ? `raw screenshot saved: ${path}` : "raw screenshot skipped: not connected"); return path; } @@ -433,7 +342,7 @@ class DashboardController { // truth while typing; echoing activeTextSettingValue back into its two-way // binding on every keystroke drops fast/pasted characters (observed: a // 51-char API key stored as its first 46 chars). Just refresh the preview. - this.previewOrRenderAfterTextSettingChange(setting.label); + this.previewOrRenderAfterTextSettingChange(); } async connect(): Promise { @@ -452,7 +361,6 @@ class DashboardController { this.lastSys = "none yet"; this.firmwareWarningMessage = ""; this.refreshBatteryOptimizationStatus(); - resetDashboardSleepTimerAndWake(); this.refreshEvenAppStatus(); this.setPhase("connecting"); this.setStatus("Connecting to the glasses..."); @@ -505,20 +413,13 @@ class DashboardController { }); }); this.offBattery = communicator.onBatteryState((state) => { - setDashboardBatteryLevels({ - headset: state.battery, - headsetCharging: state.chargingStatus > 0, - }); shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.chargingStatus > 0, }); if ((this.phase === "connected" || this.phase === "charging") && this.communicator) { + // Repaint the top bar (battery indicators live in the shell chrome). this.requestShellRender(); - void this.requestRender("interval").catch((error) => { - const message = this.formatError(error); - this.appendLog(`battery update failed: ${message}`); - }); } }); this.offEvenAppConflict = communicator.onEvenAppConflict((message) => { @@ -550,22 +451,8 @@ class DashboardController { this.emit(); } }); - this.offMedia = mediaControllerBridge.onStateChange(() => { - if (this.phase === "connected" && this.communicator) { - void this.requestRender("interval").catch((error) => { - const message = this.formatError(error); - this.appendLog(`media update failed: ${message}`); - }); - } - }); - this.offNightscout = nightscoutBridge.onStateChange(() => { - if (this.phase === "connected" && this.communicator) { - void this.requestRender("interval").catch((error) => { - const message = this.formatError(error); - this.appendLog(`nightscout update failed: ${message}`); - }); - } - }); + // The Music and Nightscout apps subscribe to their bridges directly and + // repaint their own windows, so bridge updates need no controller action. this.offVoiceStatus = voiceControlBridge.onStatus((state) => { this.appendLog(state.status); }); @@ -577,8 +464,8 @@ class DashboardController { await mediaControllerBridge.start(); await nightscoutBridge.start(); - // Register the compositor surfaces: the dashboard window in the app - // viewport, and the shell chrome above it. + // Register the compositor surfaces: the shell chrome above all windows, + // and a surface per live window (only the foreground one is composited). await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); await communicator.configureSurface(SHELL_SURFACE_ID, { x: 0, @@ -588,8 +475,6 @@ class DashboardController { zOrder: 1, transparency: "color-key", }); - // Window surfaces for every live window (including any launched while - // disconnected); only the foreground window's surface is composited. const foregroundWindowId = shell.foregroundWindow()?.windowId; for (const window of shell.getWindows()) { await this.configureWindowSurface(window.surfaceId, window.windowId === foregroundWindowId); @@ -597,25 +482,19 @@ class DashboardController { await communicator.start(); shell.foregroundWindow()?.requestRender(); this.requestShellRender(); - await this.requestRender("initial"); - this.dashboardTimer = setInterval(() => { - // Also refreshes the top-bar clock on the shell surface. + // Refresh the top-bar clock and the phone-side preview once a minute, + // and keep the Android persistent notification current. + this.shellRefreshTimer = setInterval(() => { this.requestShellRender(); - void this.requestRender("interval").catch((error) => { - const message = this.formatError(error); - this.setStatus(`Dashboard update failed: ${message}`); - this.appendLog(`dashboard update failed: ${message}`); - }); - }, DASHBOARD_INTERVAL_MS); + this.updateCompositePreview(); + this.updateConnectedForegroundNotification(); + }, SHELL_REFRESH_INTERVAL_MS); + this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS); this.screenTimeoutTimer = setInterval(() => { if (this.phase !== "connected" || !this.communicator) return; if (!shell.applyScreenTimeout()) return; this.endTextSettingEdit(); this.requestShellRender(); - void this.requestRender("interval").catch((error) => { - const message = this.formatError(error); - this.appendLog(`screen timeout render failed: ${message}`); - }); }, SCREEN_TIMEOUT_CHECK_MS); } catch (error) { const message = this.formatError(error); @@ -633,18 +512,10 @@ class DashboardController { this.offFrameMetrics = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; - this.offMedia?.(); - this.offMedia = null; - this.offNightscout?.(); - this.offNightscout = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; this.offVoiceWakeWord?.(); this.offVoiceWakeWord = null; - if (this.transcribeRenderTimer) { - clearInterval(this.transcribeRenderTimer); - this.transcribeRenderTimer = null; - } await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); @@ -681,10 +552,6 @@ class DashboardController { this.offFrameMetrics = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; - this.offMedia?.(); - this.offMedia = null; - this.offNightscout?.(); - this.offNightscout = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; this.offVoiceWakeWord?.(); @@ -706,10 +573,6 @@ class DashboardController { await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); - if (this.transcribeRenderTimer) { - clearInterval(this.transcribeRenderTimer); - this.transcribeRenderTimer = null; - } await communicator?.close().catch(() => {}); } finally { stopForegroundNotification(); @@ -754,6 +617,22 @@ class DashboardController { * gate even though the audio source is the G2 mic over BLE. */ private startVoiceCapture(): void { + this.beginVoiceCapture("ptt"); + } + + private stopVoiceCapture(): void { + voiceControlBridge.stopPushToTalk(); + } + + private startContinuousVoiceCapture(): void { + this.beginVoiceCapture("continuous"); + } + + private stopContinuousVoiceCapture(): void { + voiceControlBridge.stopContinuousCapture(); + } + + private beginVoiceCapture(kind: "ptt" | "continuous"): void { if (this.phase !== "connected" || !this.communicator) { return; } @@ -761,22 +640,23 @@ class DashboardController { void ensureVoicePermissions() .then(() => { if (this.phase !== "connected" || this.communicator !== communicator) return; - voiceControlBridge.startPushToTalk({ + const options = { communicator: communicator.getNativeCommunicator(), provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), - }); + }; + if (kind === "ptt") { + voiceControlBridge.startPushToTalk(options); + } else { + voiceControlBridge.startContinuousCapture(options); + } }) .catch((error) => { this.appendLog(`voice permission failed: ${this.formatError(error)}`); }); } - private stopVoiceCapture(): void { - voiceControlBridge.stopPushToTalk(); - } - private endTextSettingEdit(): void { const finishedSetting = this.activeTextSetting; this.activeTextSetting = null; @@ -788,23 +668,14 @@ class DashboardController { /** * Finish the active edit from the phone side (e.g. the IME's done key): - * ends the edit session and navigates the glasses out of the edit page, - * in whichever window hosts it (the Settings app or the dashboard). + * ends the edit session and navigates the Settings app's glasses editor + * out of the edit page. */ finishActiveTextSettingEdit(): void { if (!this.activeTextSetting) return; this.endTextSettingEdit(); if (this.settingsApp?.closeTextEditor()) { this.settingsApp.requestRender(); - return; - } - if (!closeDashboardTextSettingEditor()) return; - if (this.phase === "connected" && this.communicator) { - void this.requestRender("interval").catch((error) => { - this.appendLog(`edit close render failed: ${this.formatError(error)}`); - }); - } else { - this.updateDisplayPreviewFromImage(drawDashboard()); } } @@ -812,7 +683,7 @@ class DashboardController { if (setting.get() !== value) { setting.set(value); this.emit(); - this.previewOrRenderAfterTextSettingChange(setting.label); + this.previewOrRenderAfterTextSettingChange(); } } @@ -820,151 +691,13 @@ class DashboardController { await nightscoutBridge.refreshNow().catch((error) => { this.appendLog(`nightscout settings refresh failed: ${this.formatError(error)}`); }); - if (this.phase === "connected" && this.communicator) { - await this.requestRender("interval").catch((error) => { - this.appendLog(`nightscout settings render failed: ${this.formatError(error)}`); - }); - } else { - const image = drawDashboard(); - this.updateDisplayPreviewFromImage(image); - } } - private previewOrRenderAfterTextSettingChange(label: string): void { - // Echo phone-side keystrokes into the glasses editor, wherever it lives. + private previewOrRenderAfterTextSettingChange(): void { + // Echo phone-side keystrokes into the Settings app's glasses editor. if (this.settingsApp?.isTextEditorOnTop()) { this.settingsApp.requestRender(); - return; } - if (this.phase === "connected" && this.communicator) { - void this.requestRender("interval").catch((error) => { - this.appendLog(`${label} update failed: ${this.formatError(error)}`); - }); - return; - } - const image = drawDashboard(); - this.updateDisplayPreviewFromImage(image); - } - - /** - * Serialize renders; frameId identifies the input event or timer tick that - * asked for this render (one is started here for callers that don't have - * one). Ownership of the frame passes to renderDashboard and then to the - * Java side; requests that coalesce into an already-queued render finish - * immediately as discarded. When coalescing we keep the oldest waiting frame - * so measured latency reflects the worst-served request. - */ - private async requestRender(reason: "initial" | "interval", frameId?: number): Promise { - const newFrameId = frameId ?? frameTimings.startFrame(`render:${reason}`); - this.queuedRenderReason = this.queuedRenderReason === "initial" ? "initial" : reason; - if (this.renderInProgress) { - if (this.renderQueued && this.queuedFrameId > 0) { - frameTimings.finishFrame(newFrameId, `discarded: coalesced into frame#${this.queuedFrameId}`); - } else { - this.renderQueued = true; - this.queuedFrameId = newFrameId; - frameTimings.logFrame(newFrameId, "render queued behind in-progress render"); - } - return; - } - - this.renderInProgress = true; - let frameToRender = newFrameId; - try { - while (true) { - const nextReason = this.queuedRenderReason; - this.renderQueued = false; - this.queuedRenderReason = "interval"; - try { - await this.renderDashboard(nextReason, frameToRender); - } catch (error) { - frameTimings.finishFrame(frameToRender, `discarded: render failed: ${this.formatError(error)}`); - throw error; - } - if (!this.renderQueued || this.queuedFrameId <= 0) break; - frameToRender = this.queuedFrameId; - this.queuedFrameId = 0; - } - } finally { - this.renderInProgress = false; - } - } - - private async renderDashboard(reason: "initial" | "interval", frameId: number): Promise { - console.log("renderDashboard", reason); - frameTimings.logFrame(frameId, `renderDashboard start (${reason})`); - const wantFreshData = this.nextRenderWantsFreshData; - this.nextRenderWantsFreshData = false; - beginRenderPass(!wantFreshData); - const paintStartedAtMs = Date.now(); - const image = frameTimings.span(frameId, "paint", () => - frameTimings.runWithFrame(frameId, () => drawDashboard()), - ); - const paintMs = Date.now() - paintStartedAtMs; - const paintUsedStaleData = endRenderPass(); - if (paintUsedStaleData) { - frameTimings.logFrame(frameId, "painted with stale data; will schedule a fresh-data repaint"); - } - // The dashboard still paints the full lens; its window surface only shows - // the viewport-sized top-left crop (content near the right/bottom edges - // clips until the legacy layouts are resized). - const cropped = frameTimings.span(frameId, "crop", () => cropToViewport(image)); - const fingerprint = frameTimings.span(frameId, "fingerprint", () => cropped.fingerprint()); - this.lastWindowImage = cropped; - const updatePreviewAfterTransmit = this.phase === "connected"; - if (!updatePreviewAfterTransmit) { - this.updateDisplayPreviewFromImage(image); - } - if (this.communicator && this.phase === "charging") { - // Glasses are in the charging case; keep the phone preview fresh but - // send nothing. - frameTimings.finishFrame(frameId, "discarded: glasses charging"); - this.updateConnectedForegroundNotification(); - console.log("renderDashbaord finished (charging)"); - return; - } - if (this.communicator) { - if (shell.isScreenOn()) { - await this.communicator.setG2ScreenOn(true); - } - console.log("submitSurfaceFrame"); - const buffer = frameTimings.span(frameId, "to8bpp", () => cropped.to8bppBuffer()); - // The Java side owns the frame from here: it finishes it on last-packet - // ack, dedup, supersede, or timeout. - await this.communicator.submitSurfaceFrame( - DASHBOARD_WINDOW_SURFACE_ID, - buffer, - { x: 0, y: 0, width: cropped.width, height: cropped.height }, - fingerprint, - paintMs, - frameId, - ); - const outcome = await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); - if (outcome === null) { - frameTimings.logFrame(frameId, "transmit backpressure wait timed out; render loop continuing"); - } - if (!shell.isScreenOn()) { - await this.communicator.setG2ScreenOn(false); - } - if (updatePreviewAfterTransmit) { - this.updateConnectedCompositePreview(); - } - } else { - frameTimings.finishFrame(frameId, "discarded: no communicator, preview only"); - } - - if (paintUsedStaleData) { - this.nextRenderWantsFreshData = true; - void this.requestRender("interval").catch((error) => { - this.appendLog(`fresh-data repaint failed: ${this.formatError(error)}`); - }); - } - - this.updateConnectedForegroundNotification(); - if (reason === "initial") { - this.appendLog("initial dashboard image queued"); - } - console.log("renderDashboard finished"); } private async handleInputEvent(event: RawInputEvent): Promise { @@ -974,7 +707,6 @@ class DashboardController { let frameOwned = false; try { const inputEvent = rawInputEventToInputEvent(event); - logToDashboard(inputEventToString(inputEvent)); frameTimings.spanStart(frameId, "handle-input"); let outcome: ShellInputOutcome; try { @@ -988,7 +720,6 @@ class DashboardController { if (event.kind === "sys-event") { this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`; this.appendLog(`sys-event ${this.lastSys}`); - if ( event.eventType === OsEventTypeList.FOREGROUND_EXIT_EVENT || event.eventType === OsEventTypeList.ABNORMAL_EXIT_EVENT || @@ -996,19 +727,9 @@ class DashboardController { ) { this.appendLog("display state invalidated by firmware exit event"); } - if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { this.lastInput = eventName(event.eventType); } - } else if (event.kind === "text-click" && isDashboardContainerName(event.containerName)) { - if ( - event.eventType === OsEventTypeList.SCROLL_TOP_EVENT || - event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT - ) { - this.lastSys = `TEXT/${eventName(event.eventType)}`; - this.lastInput = `TEXT_${eventName(event.eventType)}`; - this.appendLog(`text-event ${this.lastSys}`); - } } if (outcome.shell) { @@ -1144,6 +865,16 @@ class DashboardController { await this.launchInProcessApp(MUSIC_WINDOW_ID, MUSIC_SURFACE_ID, createMusicAppWindow); return; } + if (appId === "transcribe") { + await this.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) => + createTranscribeAppWindow({ + ...options, + startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), + stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), + }), + ); + return; + } if (appId === "teleprompt") { await this.launchInProcessApp(TELEPROMPT_WINDOW_ID, TELEPROMPT_SURFACE_ID, (options) => createTelepromptBrowserWindow({ @@ -1268,7 +999,6 @@ class DashboardController { this.nextShellRenderWantsFreshData = true; this.requestShellRender(); } - this.lastShellImage = image; if (!this.communicator || this.phase === "charging") { frameTimings.finishFrame(frameId, "discarded: shell render with no active connection"); return; @@ -1284,24 +1014,15 @@ class DashboardController { frameId, ); await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); - this.updateConnectedCompositePreview(); + this.updateCompositePreview(); } private async handleWakeWord(keyword: string): Promise { const normalized = keyword.trim(); if (normalized.length === 0) return; this.appendLog(`wake-word detected: ${normalized}`); - const changed = resetDashboardSleepTimerAndWake(); - if (this.phase === "connected" && this.communicator) { - if (changed) { - this.requestShellRender(); - await this.requestRender("initial"); - } - return; - } - if (changed) { - const image = drawDashboard(); - this.updateDisplayPreviewFromImage(image); + if (shell.wake("sidebar")) { + this.requestShellRender(); } } @@ -1321,9 +1042,6 @@ class DashboardController { } shell.openNotificationModal(notificationKey, wokeScreen); this.requestShellRender(); - if (this.phase === "connected" && this.communicator) { - await this.requestRender("interval"); - } } private async playBuzzerSequence(payload: Uint8Array): Promise { @@ -1333,35 +1051,19 @@ class DashboardController { await this.communicator.playBuzzerSequence(payload); } - private setTranscribeRenderActive(active: boolean): void { - if (!active || this.phase !== "connected" || !this.communicator) { - if (this.transcribeRenderTimer) { - clearInterval(this.transcribeRenderTimer); - this.transcribeRenderTimer = null; - } - return; - } - if (this.transcribeRenderTimer) return; - this.transcribeRenderTimer = setInterval(() => { - void this.requestRender("interval").catch((error) => { - this.appendLog(`transcribe render failed: ${this.formatError(error)}`); - }); - }, TRANSCRIBE_RENDER_INTERVAL_MS); - } - private clearDashboardTimer(): void { - if (this.dashboardTimer) { - clearInterval(this.dashboardTimer); - this.dashboardTimer = null; + if (this.shellRefreshTimer) { + clearInterval(this.shellRefreshTimer); + this.shellRefreshTimer = null; + } + if (this.previewTimer) { + clearInterval(this.previewTimer); + this.previewTimer = null; } if (this.screenTimeoutTimer) { clearInterval(this.screenTimeoutTimer); this.screenTimeoutTimer = null; } - if (this.transcribeRenderTimer) { - clearInterval(this.transcribeRenderTimer); - this.transcribeRenderTimer = null; - } } private setPhase(phase: ConnectionPhase): void { @@ -1402,27 +1104,13 @@ class DashboardController { this.emit(); } - private updateDisplayPreviewFromImage(image: GrayImage): void { - this.setDisplayPreview(grayImageToPreviewSource(image)); - } - /** - * Phone-side preview of what is actually on the glasses: the window surface - * in its viewport with the shell chrome composited over it, mirroring the - * Java compositor. + * Phone-side preview of what is on the glasses, fetched from the Java + * compositor so it reflects every surface (chrome + whichever app is + * foreground, including worker apps the TS side never renders). Throttled + * to avoid rebuilding the bitmap faster than the phone UI needs it. */ - private buildCompositePreviewImage(): GrayImage { - const preview = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - if (this.lastWindowImage) { - preview.bitBlt(this.lastWindowImage, APP_VIEWPORT.x, APP_VIEWPORT.y); - } - if (this.lastShellImage) { - preview.bitBlt(this.lastShellImage, 0, 0, { transparentZero: true }); - } - return preview; - } - - private updateConnectedCompositePreview(): void { + private updateCompositePreview(): void { if (!this.communicator) return; const now = Date.now(); if ( @@ -1432,7 +1120,10 @@ class DashboardController { return; } this.lastConnectedPreviewUpdateAtMs = now; - this.updateDisplayPreviewFromImage(this.buildCompositePreviewImage()); + const preview = this.communicator.getCompositePreview(); + if (preview) { + this.setDisplayPreview(preview); + } } private formatError(error: unknown): string { @@ -1448,7 +1139,7 @@ class DashboardController { case "click": return { kind: "sys-event", - containerName: CONTAINER_NAME, + containerName: "", eventType: OsEventTypeList.CLICK_EVENT, eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, systemExitReasonCode: 0, @@ -1457,7 +1148,7 @@ class DashboardController { case "double-click": return { kind: "sys-event", - containerName: CONTAINER_NAME, + containerName: "", eventType: OsEventTypeList.DOUBLE_CLICK_EVENT, eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, systemExitReasonCode: 0, @@ -1466,7 +1157,7 @@ class DashboardController { case "scroll-up": return { kind: "text-click", - containerName: CONTAINER_NAME, + containerName: "", eventType: OsEventTypeList.SCROLL_TOP_EVENT, eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, systemExitReasonCode: 0, @@ -1476,7 +1167,7 @@ class DashboardController { default: return { kind: "text-click", - containerName: CONTAINER_NAME, + containerName: "", eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT, eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, systemExitReasonCode: 0, diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 7962d8c8..3e6f1ee2 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -1,8 +1,10 @@ -import { Utils } from "@nativescript/core"; +import { ImageSource, Utils } from "@nativescript/core"; import * as frameTimings from "./frame-timings"; declare const com: any; +const PREVIEW_BRIGHTEN_GAMMA = 0.7; + export type CommunicatorPhase = | "disconnected" | "connecting" @@ -338,6 +340,19 @@ export class FaceclawCommunicatorBridge { }); } + /** Phone-UI preview of the current composited screen, or null if none yet. */ + getCompositePreview(): ImageSource | null { + if (!global.isAndroid) return null; + const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA); + return bitmap ? new ImageSource(bitmap) : null; + } + + /** Save the current composite as a raw 4bpp screenshot; returns the path (empty if none). */ + saveRawCompositeScreenshot(): string { + if (!global.isAndroid) return ""; + return String(this.communicator.saveRawCompositeScreenshot()); + } + /** * Create or reconfigure a compositor surface. Geometry changes take effect * when the next frame is submitted. diff --git a/app/native/file-access.ts b/app/native/file-access.ts index fd7751b4..bea525e4 100644 --- a/app/native/file-access.ts +++ b/app/native/file-access.ts @@ -73,6 +73,30 @@ export function listDirectory(path: string): DirectoryEntry[] | null { } } +/** Write UTF-8 text to a file in the public Downloads directory; returns its path or null. */ +export function writeTextToDownloads(filename: string, text: string): string | null { + try { + const downloads = android.os.Environment.getExternalStoragePublicDirectory( + android.os.Environment.DIRECTORY_DOWNLOADS, + ); + downloads.mkdirs(); + const file = new java.io.File(downloads, filename); + const bytes = new java.lang.String(text).getBytes("UTF-8"); + // FileOutputStream rather than Files.write to avoid NativeScript varargs / + // overload resolution pitfalls. + const stream = new java.io.FileOutputStream(file); + try { + stream.write(bytes); + } finally { + stream.close(); + } + return String(file.getAbsolutePath()); + } catch (error) { + console.warn(`writeTextToDownloads failed for ${filename}: ${error}`); + return null; + } +} + /** Read a UTF-8 text file (size-capped); null when unreadable or too large. */ export function readTextFile(path: string): string | null { try { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index e7b7d09e..7190a8ef 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -27,6 +27,9 @@ export type PushToTalkOptions = { saveRecording: boolean; }; +/** Who currently wants the mic running. */ +type CaptureHolder = "ptt" | "continuous"; + export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly wakeWordListeners = new Set<(keyword: string) => void>(); @@ -35,6 +38,11 @@ export class FaceclawVoiceControlBridge { private listenerProxy: any | null = null; private status = "Voice control stopped."; private started = false; + // The mic is a single shared stream; these are the reasons it is running. + // The first holder starts capture (choosing the provider); the mic stops + // when the last one releases. Transcript events broadcast to every + // listener, so push-to-talk and the Transcribe window both receive text. + private readonly captureHolders = new Set(); // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: ElevenLabsSttClient | null = null; @@ -54,10 +62,34 @@ export class FaceclawVoiceControlBridge { return () => this.transcriptListeners.delete(listener); } - /** Begin push-to-talk capture with the selected provider. */ + /** Begin push-to-talk capture (momentary; released with stopPushToTalk). */ startPushToTalk(options: PushToTalkOptions): void { + this.acquireCapture("ptt", options); + } + + /** End push-to-talk: for cloud, commit for a final result if it was the last holder. */ + stopPushToTalk(): void { + this.releaseCapture("ptt", true); + } + + /** Begin continuous capture (Transcribe): the mic stays on until released. */ + startContinuousCapture(options: PushToTalkOptions): void { + this.acquireCapture("continuous", options); + } + + stopContinuousCapture(): void { + this.releaseCapture("continuous", false); + } + + private acquireCapture(holder: CaptureHolder, options: PushToTalkOptions): void { if (!global.isAndroid) return; - if (this.started) this.stop(); + const wasIdle = this.captureHolders.size === 0; + this.captureHolders.add(holder); + if (!wasIdle) { + // Mic already running; the new holder just shares the existing stream + // (transcripts are already broadcast to its listeners). + return; + } this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setSaveRecordings(options.saveRecording); @@ -83,21 +115,34 @@ export class FaceclawVoiceControlBridge { this.controller?.start("onboard"); } - /** End push-to-talk: stop the mic and, for cloud, commit for a final result. */ - stopPushToTalk(): void { + private releaseCapture(holder: CaptureHolder, commit: boolean): void { + if (!this.captureHolders.delete(holder)) { + // Never held; still finish a dangling cloud commit if one is pending. + if (commit) this.cloudClient?.finish(); + return; + } + if (this.captureHolders.size > 0) { + // Another holder still wants the mic; keep it running. + return; + } if (!this.started) { this.cloudClient?.finish(); return; } // Order matters for cloud: stopping the Java controller flushes any final - // decode/PCM; then commit so ElevenLabs finalizes the transcript. The - // socket is closed once the committed transcript arrives, or on stop(). + // decode/PCM; then commit so ElevenLabs finalizes the transcript. this.controller?.stop(); this.started = false; - this.cloudClient?.finish(); + if (commit) { + this.cloudClient?.finish(); + } else { + this.cloudClient?.stop(); + this.cloudClient = null; + } } stop(): void { + this.captureHolders.clear(); if (global.isAndroid) { this.controller?.stop(); } diff --git a/app/ui/apps/debug-log.ts b/app/ui/apps/debug-log.ts deleted file mode 100644 index 79a58917..00000000 --- a/app/ui/apps/debug-log.ts +++ /dev/null @@ -1,71 +0,0 @@ -import takeRight from "lodash/takeRight"; -import { getDefaultSmallFont } from "~/graphics/bdffont"; -import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin"; -import { Layer, LayerContext, type DashboardInputEvent } from "../layers"; -import { GrayImage } from "~/graphics/image"; - -export class DebugLogDashboardPlugin extends DashboardPlugin { - constructor() { - super({ - id: "input-debug-log", - label: "Input debug log", - }); - } - - override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void { - const font = getDefaultSmallFont(); - image.drawText(font, bounds.x + 10, bounds.y + 14, "Input log", 180); - const lineHeight = 14; - const visibleLineCount = Math.max(1, Math.floor((bounds.height - 28) / lineHeight)); - const visibleLines = takeRight(state.logLines, visibleLineCount); - for (let i = 0; i < visibleLines.length; i++) { - image.drawText(font, bounds.x + 8, bounds.y + 26 + i * lineHeight, visibleLines[i]!, 190); - } - } - - override createFullscreenLayer(getState: () => DashboardPluginState): Layer { - return new DebugLogLayer(getState); - } -} - -class DebugLogLayer implements Layer { - private scrollOffset = 0; - - constructor(private readonly getState: () => DashboardPluginState) {} - - paint(ctx: LayerContext): GrayImage { - const font = getDefaultSmallFont(); - const image = new GrayImage(576, 288, 0); - image.drawRect(12, 12, 552, 264, 52); - image.drawText(font, 22, 16, "Input debug log", 220); - - const logs = this.getState().logLines; - const lineHeight = 14; - const visibleCount = Math.max(1, Math.floor((288 - 64) / lineHeight)); - const maxOffset = Math.max(0, logs.length - visibleCount); - this.scrollOffset = Math.max(0, Math.min(this.scrollOffset, maxOffset)); - const start = Math.max(0, maxOffset - this.scrollOffset); - const visible = logs.slice(start, start + visibleCount); - for (let index = 0; index < visible.length; index++) { - image.drawText(font, 22, 42 + index * lineHeight, visible[index]!, 180); - } - image.drawText(font, 22, 252, "Scroll: browse Double-click: back", 110); - return image; - } - - handleInput(event: DashboardInputEvent, ctx: LayerContext): void { - switch (event.type) { - case "scroll-up": - this.scrollOffset += 1; - return; - case "scroll-down": - this.scrollOffset = Math.max(0, this.scrollOffset - 1); - return; - case "double-click": - ctx.stack.pop(); - return; - default: - return; - } - } -} \ No newline at end of file diff --git a/app/ui/apps/music-controller.ts b/app/ui/apps/music-controller.ts deleted file mode 100644 index 3601f090..00000000 --- a/app/ui/apps/music-controller.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin"; -import { getDefaultSmallFont } from "../../graphics/bdffont"; -import { GrayImage } from "../../graphics/image"; -import { wrapText } from "../../graphics/textwrap"; -import { mediaControllerBridge, type MediaControllerState } from "../../native/media-controller"; - - -export class MusicControllerDashboardPlugin extends DashboardPlugin { - constructor() { - super({ - id: "music-controller", - label: "Music controller", - }); - } - - override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void { - const font = getDefaultSmallFont(); - const media = state.media; - image.drawText(font, bounds.x + 10, bounds.y + 14, "Now playing", 180); - if (!media.accessEnabled) { - image.drawText(font, bounds.x + 10, bounds.y + 30, "Enable notification", 150); - image.drawText(font, bounds.x + 10, bounds.y + 44, "access for media", 150); - return; - } - if (!media.available) { - image.drawText(font, bounds.x + 10, bounds.y + 30, "No active session", 150); - return; - } - - const titleLines = wrapText(font, media.title || "Unknown title", bounds.width - 20).slice(0, 2); - const artistLine = media.artist || media.album || media.packageName || "Unknown source"; - image.drawText(font, bounds.x + 10, bounds.y + 30, titleLines[0] ?? "", 210); - if (titleLines[1]) { - image.drawText(font, bounds.x + 10, bounds.y + 44, titleLines[1], 210); - } - image.drawText(font, bounds.x + 10, bounds.y + 62, artistLine, 160); - image.drawText(font, bounds.x + 10, bounds.y + bounds.height - 18, playbackLabel(media), 130); - } - -} - - -function playbackLabel(media: MediaControllerState): string { - switch (media.playbackState) { - case "playing": - return "Playing"; - case "paused": - return "Paused"; - case "buffering": - return "Buffering"; - case "stopped": - return "Stopped"; - case "notification-access-required": - return "Access required"; - default: - return media.status || "Idle"; - } -} - diff --git a/app/ui/apps/nightscout.ts b/app/ui/apps/nightscout.ts index 09a92098..e5e5c31a 100644 --- a/app/ui/apps/nightscout.ts +++ b/app/ui/apps/nightscout.ts @@ -1,8 +1,8 @@ import { BdfFont, getDefaultLargeFont, getDefaultSmallFont } from "../../graphics/bdffont"; import { GrayImage, imageFromAsciiArt } from "../../graphics/image"; -import { DashboardPlugin, type DashboardPluginCardRenderArgs, type DashboardPluginState } from "../dashboard-plugin"; import { DashboardInputEvent, Layer, LayerContext } from "../layers"; import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge"; +import { isNightscoutSettingsConfigured } from "../dashboard-settings"; import { formatAgeShortFromTimestamp, formatTimestamp } from "~/util/date-util"; const nightscoutLargeFont = getDefaultLargeFont(); @@ -10,84 +10,6 @@ const NIGHTSCOUT_STALE_MS = 15 * 60 * 1000; const NIGHTSCOUT_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000; const NIGHTSCOUT_GRAPH_TIME_QUANTUM_MS = 60 * 1000; - -export class NightscoutDashboardPlugin extends DashboardPlugin { - constructor() { - super({ - id: "nightscout", - label: "Nightscout", - }); - } - - override renderCard({ image, bounds, state }: DashboardPluginCardRenderArgs): void { - const font = getDefaultSmallFont(); - const nightscout = state.nightscout; - const nowMs = Date.now(); - image.drawText(font, bounds.x + 10, bounds.y + 10, "Nightscout", 180); - if (!state.nightscoutConfigured || nightscout.configurationMissing) { - image.drawText(font, bounds.x + 10, bounds.y + 30, "Needs setup", 170); - image.drawText(font, bounds.x + 10, bounds.y + 44, "Click to configure", 140); - return; - } - if (!nightscout.available || !nightscout.latest) { - image.drawText(font, bounds.x + 10, bounds.y + 30, "No glucose data", 150); - image.drawText(font, bounds.x + 10, bounds.y + 44, truncateLine(nightscout.status, 26), 120); - return; - } - - const latest = nightscout.latest; - const deltaLabel = formatDelta(nightscout.delta); - const glucoseText = `${latest.sgv}`; - const glucoseX = bounds.x + bounds.width - 92; - image.drawText(nightscoutLargeFont, glucoseX, bounds.y + 10, glucoseText, 230); - if (isNightscoutPointStale(latest, nowMs)) { - drawNightscoutValueStrikeThrough(image, glucoseX, bounds.y + 22, nightscoutLargeFont.measureText(glucoseText)); - } - drawNightscoutDelta(image, font, bounds.x + bounds.width - 92, bounds.y + 30, deltaLabel, nightscout.direction, 160); - image.drawText( - font, - bounds.x + 10, - bounds.y + 30, - `IOB ${nightscout.iob === null ? "--" : nightscout.iob.toFixed(1)} COB ${nightscout.cob === null ? "--" : formatWholeNumber(nightscout.cob)}`, - 160, - ); - image.drawText( - font, - bounds.x + 10, - bounds.y + 44, - `Loop ${nightscout.openapsStatusShort} CAGE ${formatAgeShortFromTimestamp(nightscout.cageTimestampMs, nowMs)}`, - 138, - ); - drawNightscoutGraph( - image, - { - x: bounds.x + 10, - y: bounds.y + 60, - width: bounds.width - 20, - height: Math.max(16, bounds.height - 82), - }, - nightscout, - nowMs, - font, - ); - } - -} - -function drawNightscoutDelta( - image: GrayImage, - font: BdfFont, - x: number, - y: number, - deltaLabel: string, - direction: string, - shade: number, -): void { - const text = direction ? `${deltaLabel} ` : deltaLabel; - image.drawText(font, x, y, text, shade); - drawDirectionIndicator(image, font, x + font.measureText(text), y, direction, shade); -} - function drawDirectionIndicator( image: GrayImage, font: BdfFont, @@ -364,20 +286,18 @@ function drawNightscoutReferenceLine( export class NightscoutLayer implements Layer { - constructor(private readonly getState: () => DashboardPluginState) {} - paint(ctx: LayerContext): GrayImage { const font = getDefaultSmallFont(); // Sized to the hosting stack (the Nightscout app viewport). const { width, height } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); - const nightscout = this.getState().nightscout; + const nightscout = nightscoutBridge.snapshot(); const nowMs = Date.now(); const footerY = height - 26; image.drawRect(12, 12, width - 24, height - 24, 52); image.drawText(font, 22, 16, "Nightscout", 220); - if (!this.getState().nightscoutConfigured || nightscout.configurationMissing) { + if (!isNightscoutSettingsConfigured() || nightscout.configurationMissing) { image.drawText(font, 22, 44, "Nightscout needs configuration.", 180); image.drawText(font, 22, 62, "Use Settings > Integrations > Nightscout.", 140); return image; diff --git a/app/ui/apps/transcribe.ts b/app/ui/apps/transcribe.ts index efe10c55..1f0eb40b 100644 --- a/app/ui/apps/transcribe.ts +++ b/app/ui/apps/transcribe.ts @@ -1,50 +1,65 @@ -import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; +import { GrayImage } from "../../graphics/image"; import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; +import { writeTextToDownloads } from "../../native/file-access"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; -import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; +import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; +/** + * Live transcription view. Continuous mic capture is owned by the app wrapper + * (createTranscribeAppWindow); this layer accumulates the transcript, shows + * it, and saves it to Downloads on click. + */ export class TranscribeLayer implements Layer { - private status = "Starting transcription..."; + private status = "Listening..."; // Finalized utterances, plus the live (replace-semantics) partial appended // when painting. private finalizedText = ""; private liveText = ""; + private saveNotice = ""; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; - private actions: LayerActions | null = null; start(ctx: LayerContext): void { - this.actions = ctx.actions; - this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); + this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { + this.onTranscript(event); + ctx.actions.requestRender(); + }); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { this.status = state.status; ctx.actions.requestRender(); }); - void ctx.actions.setTranscribeRenderActive(true); - void Promise.resolve(ctx.actions.startVoiceCapture()); } - paint(): GrayImage { + paint(ctx: LayerContext): GrayImage { const font = getDefaultSmallFont(); - const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); const text = this.displayText() || "Listening..."; - const wrapped = wrapTranscribeText(font, text, G2_LENS_WIDTH - 64); + const wrapped = wrapTranscribeText(font, text, width - 64); - image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52); - image.drawText(font, 24, 24, "Transcribe", 200); - image.drawText(font, 24, 46, this.status, 110); + image.drawRect(12, 12, width - 24, height - 24, 52); + image.drawText(font, 24, 20, "Transcribe", 200); + image.drawText(font, 24, 40, this.saveNotice || this.status, 110); - const firstLine = Math.max(0, wrapped.length - 10); + const bodyTop = 62; + const footerY = height - 18; + const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / 16)); + const firstLine = Math.max(0, wrapped.length - bodyLines); for (let index = firstLine; index < wrapped.length; index++) { - const y = 72 + (index - firstLine) * 16; + const y = bodyTop + (index - firstLine) * 16; image.drawText(font, 32, y, wrapped[index]!, 230); } - image.drawText(font, 24, 252, "Double-click: back", 110); + image.drawText(font, 24, footerY, "Click: save Double-click: back", 110); return image; } handleInput(event: DashboardInputEvent, ctx: LayerContext): void { + if (event.type === "click") { + this.saveTranscript(); + ctx.actions.requestRender(); + return; + } if (event.type === "double-click") { ctx.stack.pop(); } @@ -55,9 +70,17 @@ export class TranscribeLayer implements Layer { this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; - void this.actions?.setTranscribeRenderActive(false); - void this.actions?.stopVoiceCapture(); - this.actions = null; + } + + private saveTranscript(): void { + const text = this.displayText().trim(); + if (!text) { + this.saveNotice = "Nothing to save yet."; + return; + } + const filename = `transcript-${transcriptTimestamp()}.txt`; + const path = writeTextToDownloads(filename, `${text}\n`); + this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } private displayText(): string { @@ -80,6 +103,15 @@ export class TranscribeLayer implements Layer { } } +function transcriptTimestamp(): string { + const now = new Date(); + const pad = (value: number) => String(value).padStart(2, "0"); + return ( + `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}` + + `-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}` + ); +} + function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): string[] { const words = text.split(/\s+/).filter(Boolean); const lines: string[] = []; @@ -98,4 +130,3 @@ function wrapTranscribeText(font: BdfFont, text: string, maxWidth: number): stri } return lines.length ? lines : [""]; } - diff --git a/app/ui/dashboard-plugin.ts b/app/ui/dashboard-plugin.ts deleted file mode 100644 index 47ac3bcd..00000000 --- a/app/ui/dashboard-plugin.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { type GrayImage } from "../graphics/image"; -import { type MediaControllerState } from "../native/media-controller"; -import { type NightscoutState } from "../native/nightscout-bridge"; -import { type DashboardPluginId } from "./dashboard-settings"; -import { type Layer } from "./layers"; - -export type DashboardPluginState = { - logLines: string[]; - media: MediaControllerState; - nightscout: NightscoutState; - nightscoutConfigured: boolean; -}; - -export type DashboardPluginCardBounds = { - x: number; - y: number; - width: number; - height: number; -}; - -export type DashboardPluginCardRenderArgs = { - image: GrayImage; - bounds: DashboardPluginCardBounds; - selected: boolean; - state: DashboardPluginState; -}; - -export abstract class DashboardPlugin { - readonly id: DashboardPluginId; - readonly label: string; - - constructor(config: { - id: DashboardPluginId; - label: string; - }) { - this.id = config.id; - this.label = config.label; - } - - abstract renderCard(args: DashboardPluginCardRenderArgs): void; - - createFullscreenLayer(getState: () => DashboardPluginState): Layer | null { - return null; - } -} diff --git a/app/ui/dashboard-plugins.ts b/app/ui/dashboard-plugins.ts deleted file mode 100644 index f2689745..00000000 --- a/app/ui/dashboard-plugins.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { type DashboardPluginId } from "./dashboard-settings"; -import { DashboardPlugin } from "./dashboard-plugin"; -import { NightscoutDashboardPlugin } from "./apps/nightscout"; -import { MusicControllerDashboardPlugin } from "./apps/music-controller"; -import { DebugLogDashboardPlugin } from "./apps/debug-log"; - -export { DashboardPlugin } from "./dashboard-plugin"; -export type { DashboardPluginCardBounds, DashboardPluginCardRenderArgs, DashboardPluginState } from "./dashboard-plugin"; - -let dashboardPlugins: DashboardPlugin[] | null = null; -function getDashboardPlugins(): DashboardPlugin[] { - if (!dashboardPlugins) { - dashboardPlugins = [ - { - id: "blank", - label: "Blank", - renderCard: () => {}, - createFullscreenLayer: () => null, - }, - new DebugLogDashboardPlugin(), - new MusicControllerDashboardPlugin(), - new NightscoutDashboardPlugin(), - ]; - } - return dashboardPlugins; -}; - -export function listDashboardPlugins(): DashboardPlugin[] { - return [...getDashboardPlugins()]; -} - -export function getDashboardPlugin(pluginId: DashboardPluginId): DashboardPlugin { - const PLUGIN_MAP = new Map( - getDashboardPlugins().map((plugin) => [plugin.id, plugin]), - ); - return PLUGIN_MAP.get(pluginId) ?? PLUGIN_MAP.get("blank")!; -} - -export function isBlankDashboardPlugin(pluginId: DashboardPluginId): boolean { - return pluginId === "blank"; -} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 103dfa25..eef02c1e 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -11,24 +11,11 @@ import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from import { DashboardInputEvent, Layer, type LayerContext } from "./layers"; import { GrayImage } from "~/graphics/image"; -export type DashboardSlotId = "bottom-left" | "bottom-right"; -export const dashboardSlotIds: DashboardSlotId[] = ["bottom-left", "bottom-right"]; -export type DashboardPluginId = "blank" | "input-debug-log" | "music-controller" | "nightscout"; - -export type DashboardSlotConfig = Record; export type NightscoutSettings = { siteUrl: string; apiToken: string; }; export type BatteryDisplayMode = "icon" | "percentage"; -export type SystemCardSettings = { - showFaceclawLogo: boolean; - showBatteryIndicators: boolean; - batteryDisplayMode: BatteryDisplayMode; - showAndroidNotifications: boolean; - showSignalStrength: boolean; -}; -export type SystemCardSettingKey = keyof SystemCardSettings; export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "never"; type ConfigSettingOptions = { @@ -171,35 +158,6 @@ export class ConfigSettingString extends ConfigSett } -const KNOWN_PLUGIN_IDS = new Set([ - "blank", - "input-debug-log", - "music-controller", - "nightscout", -]); - - -export const bottomLeftSlotSetting = new ConfigSettingEnum({ - id: "bottom-left-slot", - label: "Bottom left slot", - storageKey: "dashboard.slot.bottomLeft", - defaultValue: "blank", - values: Array.from(KNOWN_PLUGIN_IDS), -}); - -export const bottomRightSlotSetting = new ConfigSettingEnum({ - id: "bottom-right-slot", - label: "Bottom right slot", - storageKey: "dashboard.slot.bottomRight", - defaultValue: "blank", - values: Array.from(KNOWN_PLUGIN_IDS), -}); - -export const dashboardSlotSettings: Record> = { - "bottom-left": bottomLeftSlotSetting, - "bottom-right": bottomRightSlotSetting, -}; - export const systemCardNameSetting = new ConfigSettingString({ id: "dashboard-name", label: "Dashboard name", @@ -211,20 +169,6 @@ export const systemCardNameSetting = new ConfigSettingString({ formatValue: (value) => value || "Faceclaw", }); -export const showFaceclawLogoSetting = new ConfigSettingBoolean({ - id: "showFaceclawLogo", - label: "Show Faceclaw Logo", - storageKey: "dashboard.systemCard.showFaceclawLogo", - defaultValue: true, -}); - -export const showBatteryIndicatorsSetting = new ConfigSettingBoolean({ - id: "showBatteryIndicators", - label: "Show Battery Indicators", - storageKey: "dashboard.systemCard.showBatteryIndicators", - defaultValue: true, -}); - export const batteryDisplayModeSetting = new ConfigSettingEnum({ id: "batteryDisplayMode", label: "Battery display", @@ -234,20 +178,6 @@ export const batteryDisplayModeSetting = new ConfigSettingEnum({ id: "screen-timeout", label: "Screen timeout", @@ -388,15 +318,6 @@ export function isNightscoutSettingsConfigured(): boolean { return nightscoutSiteUrlSetting.get().length > 0 && nightscoutApiTokenSetting.get().length > 0; } -export function normalizePluginId( - value: string | null | undefined, - fallback: DashboardPluginId = "blank", -): DashboardPluginId { - if (value && KNOWN_PLUGIN_IDS.has(value as DashboardPluginId)) { - return value as DashboardPluginId; - } - return fallback; -} function normalizeSystemCardName(name: string | null | undefined): string { const normalized = (name ?? "").replace(/[\x00-\x1f]+/g, " ").replace(/\s+/g, " ").trim(); diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts deleted file mode 100644 index a069ef7a..00000000 --- a/app/ui/dashboard.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; -import { mediaControllerBridge } from "../native/media-controller"; -import { nightscoutBridge } from "../native/nightscout-bridge"; -import { getDashboardPlugin, type DashboardPluginCardBounds, type DashboardPluginState } from "./dashboard-plugins"; -import { EditTextSettingLayer, isNightscoutSettingsConfigured, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings"; -import { Layer, LayerActions, LayerStack, type DashboardInputEvent } from "./layers"; -import { drawSystemCard } from "./dashboard/system-card"; -import { createRootMenuLayer } from "./dashboard/root-menu"; -import { inputEventToString, shell } from "./shell/shell"; - -type DashboardCardId = "system" | DashboardSlotId; -export type DashboardBatteryLevels = { - headset: number | null; - headsetCharging: boolean | null; - ring: number | null; -}; - -type DashboardState = { - logLines: string[]; - battery: DashboardBatteryLevels; -}; - -export let dashboardState: DashboardState = { - logLines: [] as string[], - battery: { - headset: null, - headsetCharging: null, - ring: null, - } as DashboardBatteryLevels, -}; - -const dashboardActions: LayerActions = { - requestRender: () => {}, - disconnect: () => {}, - startTextSettingEdit: () => {}, - endTextSettingEdit: () => {}, - setTranscribeRenderActive: () => {}, - startVoiceCapture: () => {}, - stopVoiceCapture: () => {}, - playBuzzerSequence: () => {}, -}; -export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 }; - -/** - * Input handler for the dashboard window. Global gestures (wake, sleep, - * long-press/push-to-talk) are consumed by the shell before this is called. - */ -export async function receiveDashboardWindowInput(inputEvent: DashboardInputEvent): Promise { - dashboardState.logLines.push(inputEventToString(inputEvent)); - await dashboardLayers.handleInput(inputEvent); -} - -export function logToDashboard(message: string): void { - dashboardState.logLines.push(message); -} - -export function setDashboardActions(actions: Partial): void { - dashboardLayers.setActions(actions); -} - -/** - * Close the glasses-side text-setting edit page, e.g. when the user finishes - * typing on the phone. Returns whether the display changed. - */ -export function closeDashboardTextSettingEditor(): boolean { - return dashboardLayers.popIfTop((layer) => layer instanceof EditTextSettingLayer); -} - -export function drawDashboard(): GrayImage { - if (!shell.isScreenOn()) { - return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - } - return dashboardLayers.paint(); -} - -export function resetDashboardSleepTimerAndWake(nowMs = Date.now()): boolean { - if (!shell.wake("sidebar", nowMs)) return false; - dashboardLayers.clearToBase(); - dashboardLayers.push(createRootMenuLayer()); - return true; -} - -export function setDashboardBatteryLevels(levels: Partial): void { - dashboardState.battery = { - ...dashboardState.battery, - ...levels, - }; -} - -export function getPluginIdForSlot(slot: DashboardSlotId): DashboardPluginId { - switch (slot) { - case "bottom-left": - return bottomLeftSlotSetting.get(); - case "bottom-right": - return bottomRightSlotSetting.get(); - } -} - - -class DashboardLayer implements Layer { - paint(): GrayImage { - const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - const pluginState = getPluginState(); - drawSystemCard(image, getCardBounds("system")); - - for (const slot of dashboardSlotIds) { - const bounds = getCardBounds(slot); - const pluginId = getPluginIdForSlot(slot); - getDashboardPlugin(pluginId).renderCard({ - image, - bounds, - selected: false, - state: pluginState, - }); - } - - return image; - } - - handleInput(event: DashboardInputEvent, ctx: { stack: LayerStack }): void { - if (event.type === "click") { - ctx.stack.push(createRootMenuLayer()); - return; - } - - if (event.type === "double-click") { - // Backing out of the window's root returns focus to the shell sidebar. - shell.yieldFocusToSidebar(); - } - } -} - -export function getPluginState(): DashboardPluginState { - return { - logLines: dashboardState.logLines, - media: mediaControllerBridge.snapshot(), - nightscout: nightscoutBridge.snapshot(), - nightscoutConfigured: isNightscoutSettingsConfigured(), - }; -} - -function getCardBounds(card: DashboardCardId): DashboardPluginCardBounds { - switch (card) { - case "system": - return { - x: G2_LENS_WIDTH / 2 + 2, - y: 0, - width: G2_LENS_WIDTH / 2 - 2, - height: G2_LENS_HEIGHT / 2 - 2, - }; - case "bottom-left": - return { - x: 0, - y: G2_LENS_HEIGHT / 2 + 2, - width: G2_LENS_WIDTH / 2 - 2, - height: G2_LENS_HEIGHT / 2 - 2, - }; - case "bottom-right": - return { - x: G2_LENS_WIDTH / 2 + 2, - y: G2_LENS_HEIGHT / 2 + 2, - width: G2_LENS_WIDTH / 2 - 2, - height: G2_LENS_HEIGHT / 2 - 2, - }; - } -} - -const dashboardLayers = new LayerStack(new DashboardLayer(), dashboardActions); -dashboardLayers.push(createRootMenuLayer()); - diff --git a/app/ui/dashboard/root-menu.ts b/app/ui/dashboard/root-menu.ts deleted file mode 100644 index ce975e2c..00000000 --- a/app/ui/dashboard/root-menu.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { getDefaultSmallFont } from "~/graphics/bdffont"; -import { getPluginIdForSlot, getPluginState, TOP_LEFT_MENU_LAYOUT } from "../dashboard"; -import { getDashboardPlugin, isBlankDashboardPlugin } from "../dashboard-plugins"; -import { DashboardSlotId, dashboardSlotIds, isNightscoutSettingsConfigured } from "../dashboard-settings"; -import { DashboardInputEvent, LayerContext } from "../layers"; -import { MenuLayer } from "../menu"; -import { createNightscoutSettingsMenuLayer } from "./settings-menus"; -import { TranscribeLayer } from "../apps/transcribe"; -import { GrayImage } from "~/graphics/image"; -import { shell } from "../shell/shell"; - -const rootMenuFont = getDefaultSmallFont(); - -class RootMenuLayer extends MenuLayer { - async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { - if (event.type === "double-click") { - // Backing out of the window's root returns focus to the shell sidebar. - shell.yieldFocusToSidebar(); - return; - } - await super.handleInput(event, ctx); - } -} - -export function createRootMenuLayer(): MenuLayer { - return new RootMenuLayer( - null, - [ - { - label: "Apps", - onSelect: (ctx) => { - ctx.stack.push(createAppsMenuLayer()); - }, - }, - ...dashboardSlotIds - .filter((slot) => !isBlankDashboardPlugin(getPluginIdForSlot(slot))) - .map((slot) => ({ - label: pluginLabelForSlot(slot), - onSelect: (ctx: LayerContext) => { - openDashboardSlot(slot, ctx); - }, - render: ({ image, x, y }: { image: GrayImage; x: number; y: number }) => { - image.drawText(rootMenuFont, x, y + 3, pluginLabelForSlot(slot), 200); - }, - })), - ], - TOP_LEFT_MENU_LAYOUT, - ); -} - -function createAppsMenuLayer(): MenuLayer { - return new MenuLayer( - "Apps", - [ - { - label: "Transcribe", - onSelect: (ctx) => { - const layer = new TranscribeLayer(); - ctx.stack.push(layer); - layer.start(ctx); - }, - }, - ], - TOP_LEFT_MENU_LAYOUT, - ); -} - -function openDashboardSlot(slot: DashboardSlotId, ctx: LayerContext): void { - const pluginId = getPluginIdForSlot(slot); - if (isBlankDashboardPlugin(pluginId)) { - return; - } - if (pluginId === "nightscout" && !isNightscoutSettingsConfigured()) { - ctx.stack.push(createNightscoutSettingsMenuLayer()); - return; - } - const layer = getDashboardPlugin(pluginId).createFullscreenLayer?.(getPluginState); - if (layer) { - ctx.stack.push(layer); - } -} - - -function pluginLabelForSlot(slot: DashboardSlotId): string { - return getDashboardPlugin(getPluginIdForSlot(slot)).label; -} diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 189dd1b4..384b9dad 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -1,14 +1,12 @@ import { APP_VIEWPORT } from "../shell/geometry"; -import { type MenuLayout } from "../menu"; - -// Settings menus render in both the dashboard window (full lens) and the -// Settings app window (viewport-sized); cap heights to the smaller of the two. -const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 }; +import { MenuLayer, type MenuLayout } from "../menu"; import { shell } from "../shell/shell"; -import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings"; -import { MenuLayer } from "../menu"; +import { batteryDisplayModeSetting, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, terminalAuthTokenSetting, terminalHostSetting, terminalPortSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings"; import { AboutPage } from "./about-page"; +// Rendered in the Settings app window (viewport-sized); cap height to it. +const SETTINGS_MENU_LAYOUT: MenuLayout = { x: 8, y: 8, width: 272, maxHeight: APP_VIEWPORT.height - 16 }; + export function createSettingsMenuLayer(): MenuLayer { return new MenuLayer( "Settings", @@ -19,12 +17,6 @@ export function createSettingsMenuLayer(): MenuLayer { ctx.stack.push(createDisplaySettingsMenuLayer()); }, }, - { - label: "Dashboard", - onSelect: (ctx) => { - ctx.stack.push(createDashboardSettingsMenuLayer()); - }, - }, { label: "Voice", onSelect: (ctx) => { @@ -96,10 +88,12 @@ function createDisplaySettingsMenuLayer(): MenuLayer { "Settings > Display", [ enumSettingMenuItem(screenTimeoutSetting, { - onChange: (ctx, newValue, oldValue) => { + onChange: () => { shell.noteUserActivity(); }, }), + // Controls the top-bar battery indicators (icon vs percentage). + enumSettingMenuItem(batteryDisplayModeSetting), ], SETTINGS_MENU_LAYOUT, ); @@ -116,39 +110,6 @@ function createVoiceSettingsMenuLayer(): MenuLayer { ); } -function createDashboardSettingsMenuLayer(): MenuLayer { - return new MenuLayer( - "Settings > Dashboard", - [ - { - label: "System Card", - onSelect: (ctx) => { - ctx.stack.push(createSystemCardSettingsMenuLayer()); - }, - }, - ...dashboardSlotIds.map((slot) => enumSettingMenuItem(dashboardSlotSettings[slot], { - style: "submenu", - })), - ], - SETTINGS_MENU_LAYOUT, - ); -} - -function createSystemCardSettingsMenuLayer(): MenuLayer { - return new MenuLayer( - "Settings > Dashboard > System Card", - [ - textSettingMenuItem(systemCardNameSetting), - toggleSettingMenuItem(showFaceclawLogoSetting), - toggleSettingMenuItem(showBatteryIndicatorsSetting), - enumSettingMenuItem(batteryDisplayModeSetting), - toggleSettingMenuItem(showAndroidNotificationsSetting), - toggleSettingMenuItem(showSignalStrengthSetting), - ], - SETTINGS_MENU_LAYOUT, - ); -} - function createIntegrationsMenuLayer(): MenuLayer { return new MenuLayer( "Settings > Integrations", diff --git a/app/ui/dashboard/system-card.ts b/app/ui/dashboard/system-card.ts deleted file mode 100644 index cdc7fa7e..00000000 --- a/app/ui/dashboard/system-card.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { GrayImage } from "../../graphics/image"; -import { spanCurrent } from "../../native/frame-timings"; -import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness"; -import { BATTERY_ICON_WIDTH, drawBattery } from "../../graphics/battery"; -import { readActiveNotificationIcons } from "../../native/notification-icons"; -import { readPhoneBatteryState } from "../../native/phone-battery"; -import { readSystemStatusIcons } from "../../native/system-status-icons"; -import { type DashboardPluginCardBounds } from "../dashboard-plugins"; -import { dashboardState } from "../dashboard"; -import { getDefaultMediumFont, getDefaultSmallFont } from "~/graphics/bdffont"; -import { getDashboardLogo } from "~/graphics/logo"; -import { batteryDisplayModeSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, showSignalStrengthSetting, systemCardNameSetting } from "../dashboard-settings"; - -const NOTIFICATION_ICON_SIZE = 24; -const SYSTEM_CARD_ITEM_HEIGHT = 38; -const SYSTEM_CARD_ITEM_GAP = 2; -const BATTERY_ITEM_Y_OFFSET = 4; -const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; -const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; - -type SystemCardFlowItem = - | { type: "battery"; label: string; percentCharge: number; isCharging: boolean } - | { type: "notification"; icon: GrayImage }; - -const smallFont = getDefaultSmallFont(); -const mediumFont = getDefaultMediumFont(); - -function drawSystemCardFlowItems(image: GrayImage, bounds: DashboardPluginCardBounds): void { - const left = bounds.x + 10; - const top = bounds.y + 84; - const right = bounds.x + bounds.width - 10; - const bottom = bounds.y + bounds.height - 6; - const items: SystemCardFlowItem[] = []; - if (showAndroidNotificationsSetting.get()) { - const maxNotificationIcons = Math.max(0, ((right - left) / Math.max(1, NOTIFICATION_ICON_SIZE + SYSTEM_CARD_ITEM_GAP)) | 0) * 2; - const { icons, stale } = readActiveNotificationIcons(maxNotificationIcons, renderPassAllowsStaleData()); - if (stale) { - noteStaleDataUsed(); - } - for (const icon of icons) { - items.push({ type: "notification", icon }); - } - } - items.push(...collectBatteryItems()); - if (showSignalStrengthSetting.get()) { - for (const icon of readSystemStatusIcons()) { - items.push({ type: "notification", icon }); - } - } - - let itemX = left; - let itemY = top; - for (const item of items) { - const itemWidth = systemCardFlowItemWidth(item); - if (itemX > left && itemX + itemWidth > right) { - itemX = left; - itemY += SYSTEM_CARD_ITEM_HEIGHT + SYSTEM_CARD_ITEM_GAP; - } - if (itemY + SYSTEM_CARD_ITEM_HEIGHT > bottom) { - break; - } - if (item.type === "battery") { - drawBatteryFlowItem(image, itemX, itemY, item); - } else { - image.bitBlt( - item.icon, - itemX, - itemY + ((SYSTEM_CARD_ITEM_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0, - ); - } - itemX += itemWidth + SYSTEM_CARD_ITEM_GAP; - } -} - -function systemCardFlowItemWidth(item: SystemCardFlowItem): number { - if (item.type === "notification") { - return NOTIFICATION_ICON_SIZE; - } - if (batteryDisplayModeSetting.get() === "percentage") { - const percentText = formatBatteryPercent(item.percentCharge); - return Math.max(smallFont.measureText(percentText), smallFont.measureText(item.label)); - } - return Math.max(BATTERY_ICON_WIDTH, smallFont.measureText(item.label)); -} - -function collectBatteryItems(): SystemCardFlowItem[] { - if (!showBatteryIndicatorsSetting.get()) return []; - const phone = readPhoneBatteryState(); - const items: SystemCardFlowItem[] = []; - addBatteryItem(items, "Phone", phone.battery, phone.charging); - addBatteryItem(items, "G2", dashboardState.battery.headset, dashboardState.battery.headsetCharging); - addBatteryItem(items, "R1", dashboardState.battery.ring, null); - return items; -} - -function addBatteryItem( - items: SystemCardFlowItem[], - label: string, - percentCharge: number | null, - isCharging: boolean | null, -): void { - if (percentCharge === null || !Number.isFinite(percentCharge)) return; - items.push({ - type: "battery", - label, - percentCharge, - isCharging: Boolean(isCharging), - }); -} - -function formatBatteryPercent(percentCharge: number): string { - const clamped = Math.max(0, Math.min(100, Math.round(percentCharge))); - return `${clamped}%`; -} - -function drawBatteryFlowItem(image: GrayImage, x: number, y: number, item: Extract): void { - const itemWidth = systemCardFlowItemWidth(item); - const labelX = x + Math.max(0, ((itemWidth - smallFont.measureText(item.label)) / 2) | 0); - image.drawText(smallFont, labelX, y + BATTERY_ITEM_Y_OFFSET, item.label, 150); - if (batteryDisplayModeSetting.get() === "percentage") { - drawBatteryPercentFlowItem(image, x, y, item, itemWidth); - return; - } - const battery = drawBattery(item.percentCharge, item.isCharging); - image.bitBlt(battery, x + ((itemWidth - battery.width) / 2) | 0, y + 16 + BATTERY_ITEM_Y_OFFSET); -} - -function drawBatteryPercentFlowItem( - image: GrayImage, - x: number, - y: number, - item: Extract, - itemWidth: number, -): void { - const percentText = formatBatteryPercent(item.percentCharge); - const textWidth = smallFont.measureText(percentText); - const textX = x + Math.max(0, ((itemWidth - textWidth) / 2) | 0); - const textY = y + 16 + BATTERY_ITEM_Y_OFFSET; - if (item.isCharging) { - const padX = 2; - const padY = 1; - image.fillRect(textX - padX, textY - padY, textWidth + padX * 2, smallFont.lineHeight + padY * 2, 255); - image.drawText(smallFont, textX, textY, percentText, 0); - return; - } - image.drawText(smallFont, textX, textY, percentText, 200); -} - -function formatDashboardDate(now: Date): string { - return `${WEEKDAYS[now.getDay()]} ${MONTHS[now.getMonth()]} ${now.getDate()} ${pad2(now.getHours())}:${pad2(now.getMinutes())}`; -} - -function pad2(value: number): string { - return String(value).padStart(2, "0"); -} - -export function drawSystemCard(image: GrayImage, bounds: DashboardPluginCardBounds): void { - spanCurrent("draw-system-card", () => { - const now = new Date(); - const logo = getDashboardLogo(); - - if (logo && showFaceclawLogoSetting.get()) { - image.bitBlt(logo, bounds.x + 10, bounds.y + 10); - } - const infoX = logo && showFaceclawLogoSetting.get() ? bounds.x + 92 : bounds.x + 10; - image.drawText(mediumFont, infoX, bounds.y + 10, systemCardNameSetting.get(), 200); - image.drawText(mediumFont, infoX, bounds.y + 32, formatDashboardDate(now), 200); - drawSystemCardFlowItems(image, bounds); - }); -} - diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 21ba6dc1..b1ea5af9 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -18,11 +18,13 @@ export type LayerActions = { disconnect: () => Promise | void; startTextSettingEdit: (setting: ConfigSettingString) => Promise | void; endTextSettingEdit: () => Promise | void; - setTranscribeRenderActive: (active: boolean) => Promise | void; - /** Start voice capture with the provider chosen in settings. */ + /** Start push-to-talk voice capture with the provider chosen in settings. */ startVoiceCapture: () => Promise | void; - /** Stop voice capture; for a cloud provider this also commits for a final result. */ + /** Stop push-to-talk; for a cloud provider this also commits for a final result. */ stopVoiceCapture: () => Promise | void; + /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ + startContinuousVoiceCapture: () => Promise | void; + stopContinuousVoiceCapture: () => Promise | void; /** Play a CFW tone-sequencer payload (see sound-effects.ts). */ playBuzzerSequence: (payload: Uint8Array) => Promise | void; }; diff --git a/app/ui/shell/nightscout-app.ts b/app/ui/shell/nightscout-app.ts index abd649d1..58164429 100644 --- a/app/ui/shell/nightscout-app.ts +++ b/app/ui/shell/nightscout-app.ts @@ -1,7 +1,6 @@ import { getDefaultSmallFont } from "../../graphics/bdffont"; import { GrayImage } from "../../graphics/image"; import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge"; -import { getPluginState } from "../dashboard"; import { NightscoutLayer } from "../apps/nightscout"; import { createInProcessWindow, @@ -35,7 +34,7 @@ export function createNightscoutAppWindow(options: InProcessAppOptions): InProce iconLetter: "Ns", closeable: true, actions: options.actions, - baseLayer: new YieldAtRootLayer(new NightscoutLayer(getPluginState)), + baseLayer: new YieldAtRootLayer(new NightscoutLayer()), submitFrame: options.submitFrame, setSurfaceVisible: options.setSurfaceVisible, removeSurface: options.removeSurface, diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 185b0bac..b7c3a6f7 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -66,9 +66,10 @@ const noopActions: LayerActions = { disconnect: () => {}, startTextSettingEdit: () => {}, endTextSettingEdit: () => {}, - setTranscribeRenderActive: () => {}, startVoiceCapture: () => {}, stopVoiceCapture: () => {}, + startContinuousVoiceCapture: () => {}, + stopContinuousVoiceCapture: () => {}, playBuzzerSequence: () => {}, }; diff --git a/app/ui/shell/transcribe-app.ts b/app/ui/shell/transcribe-app.ts new file mode 100644 index 00000000..4bbb2dd9 --- /dev/null +++ b/app/ui/shell/transcribe-app.ts @@ -0,0 +1,77 @@ +import { imageFromAsciiArt } from "../../graphics/image"; +import { TranscribeLayer } from "../apps/transcribe"; +import type { LayerActions } from "../layers"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "./in-process-window"; +import { shell } from "./shell"; + +export const TRANSCRIBE_WINDOW_ID = "transcribe"; +export const TRANSCRIBE_SURFACE_ID = "window:transcribe"; + +const TRAY_ICON_ID = "transcribe"; + +// 16x18 microphone glyph for the top-bar tray. +const MIC_ICON = imageFromAsciiArt( + [ + " #### ", + " ###### ", + " # # ", + " # # ", + " # # ", + " # # ", + " # # ", + " # # # # ", + " # # # # ", + " # #### # ", + " # # ", + " ###### ", + " # ", + " # ", + " ###### ", + ], + 220, +); + +export type TranscribeAppOptions = InProcessAppOptions & { + /** Begin continuous mic capture (kept on while the window is open). */ + startContinuousVoiceCapture: () => void; + stopContinuousVoiceCapture: () => void; +}; + +/** + * The Transcribe app: live speech-to-text in its own window. While open it + * holds continuous mic capture (so push-to-talk can share the stream) and + * shows a microphone tray icon in the top bar. + */ +export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow { + const layer = new TranscribeLayer(); + const actions: LayerActions = { ...options.actions }; + const app = createInProcessWindow({ + appId: "transcribe", + windowId: TRANSCRIBE_WINDOW_ID, + title: "Transcribe", + iconLetter: "Tr", + closeable: true, + actions, + baseLayer: new YieldAtRootLayer(layer), + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + onClosed: () => { + options.stopContinuousVoiceCapture(); + shell.setTrayIcon(TRAY_ICON_ID, null); + options.onClosed(); + }, + }); + // start() wires the layer's transcript/status subscriptions; the LayerStack + // exposes its context through a paint, but the layer needs it up front, so + // build a minimal context here. + layer.start({ stack: app.stack, actions }); + options.startContinuousVoiceCapture(); + shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); + return app; +}