diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 5ec0fc34..9b6c9848 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -353,6 +353,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre contentFingerprint, paintMs, frameId, if (glyphs == null) null else AndroidByteReader(glyphs)) } + /** Whether submitting this full-surface content would change nothing (see [GlassesSessionCore.isSurfaceCurrent]). */ + fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean = core.isSurfaceCurrent(surfaceId, fingerprint) + /** Play a tone sequence via CFW load_image_z mode 5 kind 4 (payload built on the TS side). */ fun playBuzzerSequence(payload: java.nio.ByteBuffer?) { val bytes = ByteArray(if (payload == null) 0 else payload.remaining()) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt index 19d5a5a0..7e5cad69 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt @@ -1,14 +1,24 @@ package com.faceclaw.app import android.app.Activity +import android.content.Context import android.content.MutableContextWrapper +import android.graphics.Color +import android.graphics.Typeface import android.os.Build import android.os.Handler import android.os.Looper +import android.text.TextUtils +import android.util.TypedValue +import android.view.Gravity import android.view.View import android.view.ViewGroup import android.webkit.WebView import android.widget.FrameLayout +import android.widget.LinearLayout +import android.widget.TextView +import androidx.core.view.ViewCompat +import androidx.core.view.WindowInsetsCompat import java.util.ArrayList import java.util.HashMap @@ -34,6 +44,12 @@ import java.util.HashMap * the chosen WebView to the top of the overlay); hiding sends it back behind. * All view work happens on the main thread (callers are on the NS/JS thread). * + * Layout: the activity is edge-to-edge, so the overlay tracks the window + * insets itself. A top bar (the shown app's name and a close button) sits + * under the status bar, and the WebViews fill the area between it and the + * navigation bar / keyboard. That area is the same whether or not the overlay + * is shown, so showing an app doesn't resize (and re-lay-out) its page. + * * Timer keep-alive: when the phone screen turns off, Chromium heavily throttles * the page's own setTimeout/setInterval (intensive background throttling clamps * them to ~1/sec), so timer-driven apps (e.g. snake's game loop) slow to a @@ -71,8 +87,13 @@ import java.util.HashMap */ class FaceclawEvenHubWebViewHost { private var overlay: FrameLayout? = null + /** Holds the WebViews, inside [overlay] below the top bar. */ + private var stack: FrameLayout? = null + private var titleView: TextView? = null private var overlayActivity: Activity? = null private var shown = false + private var shownTitle = "" + private var onClose: Runnable? = null private val mainHandler = Handler(Looper.getMainLooper()) private val webViews: MutableList = ArrayList() @@ -139,6 +160,12 @@ class FaceclawEvenHubWebViewHost { private const val TICK_JS = "window.__fcTick?__fcTick():null" + /** Top bar height, below the status bar (Material toolbar height). */ + private const val BAR_HEIGHT_DP = 56f + + /** Matches the NativeScript ActionBar (NativeScriptToolbarStyle). */ + private val BAR_COLOR = Color.parseColor("#424242") + @JvmStatic @Synchronized fun getInstance(): FaceclawEvenHubWebViewHost { @@ -152,36 +179,112 @@ class FaceclawEvenHubWebViewHost { val existing = overlay if (existing != null && overlayActivity === activity && existing.parent != null) return existing val content = activity.findViewById(android.R.id.content) + (existing?.parent as ViewGroup?)?.removeView(existing) val created = FrameLayout(activity) - if (existing != null) { - (existing.parent as ViewGroup?)?.removeView(existing) - for (web in webViews) { - existing.removeView(web) - rebaseContext(web, activity) - created.addView(web, FrameLayout.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) - } + created.setBackgroundColor(Color.BLACK) + val createdStack = FrameLayout(activity) + created.addView(createdStack, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + val bar = buildTopBar(activity) + created.addView(bar, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT, Gravity.TOP)) + for (web in webViews) { + (web.parent as ViewGroup?)?.removeView(web) + rebaseContext(web, activity) + createdStack.addView(web, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + } + // Insets are passed on unconsumed: before API 30 a consuming child + // would starve its later siblings (the NativeScript UI) of them. + ViewCompat.setOnApplyWindowInsetsListener(created) { _, insets -> + applyInsets(bar, createdStack, insets) + insets } overlay = created + stack = createdStack overlayActivity = activity // index 0 = behind the NativeScript content view (raised again below if shown). content.addView(created, 0, FrameLayout.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + ViewCompat.requestApplyInsets(created) if (shown) created.bringToFront() return created } + /** The bar across the top of a shown app: the app's name, then a close button. */ + private fun buildTopBar(activity: Activity): LinearLayout { + val bar = LinearLayout(activity) + bar.orientation = LinearLayout.HORIZONTAL + bar.gravity = Gravity.CENTER_VERTICAL + bar.setBackgroundColor(BAR_COLOR) + // Swallow touches so they don't fall through to the WebViews. + bar.isClickable = true + + val title = TextView(activity) + title.text = shownTitle + title.setTextColor(Color.WHITE) + title.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + title.typeface = Typeface.create("sans-serif-medium", Typeface.NORMAL) + title.maxLines = 1 + title.ellipsize = TextUtils.TruncateAt.END + bar.addView(title, LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f).apply { + marginStart = dp(activity, 16f) + marginEnd = dp(activity, 12f) + }) + + val close = TextView(activity) + close.text = "\u2715" + close.setTextColor(Color.WHITE) + close.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + close.gravity = Gravity.CENTER + close.contentDescription = "Close" + val ripple = TypedValue() + if (activity.theme.resolveAttribute(android.R.attr.selectableItemBackgroundBorderless, ripple, true)) { + close.setBackgroundResource(ripple.resourceId) + } + close.setOnClickListener { onClose?.run() } + val closeSize = dp(activity, 48f) + bar.addView(close, LinearLayout.LayoutParams(closeSize, closeSize).apply { + marginEnd = dp(activity, 4f) + }) + titleView = title + return bar + } + + /** + * Put the bar under the status bar (its background filling in behind it) + * and the WebViews between the bar and the navigation bar or keyboard. + */ + private fun applyInsets(bar: View, webStack: View, insets: WindowInsetsCompat) { + val bars = insets.getInsets( + WindowInsetsCompat.Type.systemBars() or WindowInsetsCompat.Type.displayCutout()) + val ime = insets.getInsets(WindowInsetsCompat.Type.ime()) + val barHeight = dp(bar.context, BAR_HEIGHT_DP) + bar.setPadding(bars.left, bars.top, bars.right, 0) + bar.layoutParams = (bar.layoutParams as FrameLayout.LayoutParams).apply { + height = bars.top + barHeight + } + webStack.layoutParams = (webStack.layoutParams as FrameLayout.LayoutParams).apply { + setMargins(bars.left, bars.top + barHeight, bars.right, maxOf(bars.bottom, ime.bottom)) + } + } + + private fun dp(context: Context, value: Float): Int { + return TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, value, context.resources.displayMetrics).toInt() + } + private fun rebaseContext(web: WebView, activity: Activity) { (web.context as? MutableContextWrapper)?.baseContext = activity } /** Add a WebView to the host, full-size and rendering, hidden behind the UI. */ fun attach(activity: Activity, web: WebView) { - val o = ensureOverlay(activity) + ensureOverlay(activity) rebaseContext(web, activity) web.visibility = View.VISIBLE if (web.parent == null) { - o.addView(web, FrameLayout.LayoutParams( + stack!!.addView(web, FrameLayout.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) } // resumeTimers() is process-global; make sure nothing left timers paused. @@ -215,9 +318,15 @@ class FaceclawEvenHubWebViewHost { } } - /** Bring an app's WebView to the front of [activity] so it is visible on the phone. */ - fun showOnPhone(activity: Activity?, web: WebView) { + /** + * Bring an app's WebView to the front of [activity] so it is visible on the + * phone, titled [title]. The top bar's close button runs [onClose]. + */ + fun showOnPhone(activity: Activity?, web: WebView, title: String, onClose: Runnable) { val o = if (activity != null) ensureOverlay(activity) else overlay ?: return + shownTitle = title + this.onClose = onClose + titleView?.text = title web.bringToFront() o.bringToFront() shown = true @@ -226,6 +335,7 @@ class FaceclawEvenHubWebViewHost { /** Send the overlay back behind the NativeScript UI. */ fun hideOnPhone() { shown = false + onClose = null val o = overlay ?: return val content = o.parent as ViewGroup? if (content != null) { @@ -242,7 +352,7 @@ class FaceclawEvenHubWebViewHost { fun detach(web: WebView) { webViews.remove(web) tickers.remove(web)?.stop() - overlay?.removeView(web) + stack?.removeView(web) web.destroy() } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt index 5be4a511..fcccfb04 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt @@ -23,6 +23,8 @@ import java.nio.charset.StandardCharsets import java.util.ArrayDeque import java.util.ArrayList import java.util.Collections +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors /** * Phone side of the Wear OS integration: the single place that talks to the @@ -114,6 +116,16 @@ class FaceclawWearBridge private constructor(context: Context) { private val pending = ArrayDeque() private val watchNodes: MutableList = ArrayList() private var lastPublishedState: String? = null + /** Latest state waiting for [publishExecutor]; null when none is queued. Guarded by [lock]. */ + private var queuedState: String? = null + /** + * Serializing a DataMap and handing it to Play services costs ~10-25ms per + * publish, and the switcher publishes on every foreground change, so it + * runs here instead of on the main thread. + */ + private val publishExecutor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "FaceclawWearPublish").apply { isDaemon = true } + } private class PendingMessage( val path: String, @@ -210,12 +222,28 @@ class FaceclawWearBridge private constructor(context: Context) { publishState(json, false) } - /** As [publishState]; `force` re-sends an unchanged state. */ + /** + * As [publishState]; `force` re-sends an unchanged state. The put happens + * on [publishExecutor]; states that arrive while one is still waiting + * replace it, since only the latest matters to the watch. + */ fun publishState(json: String?, force: Boolean) { if (!available || json == null) return synchronized(lock) { if (!force && json == lastPublishedState) return lastPublishedState = json + val alreadyQueued = queuedState != null + queuedState = json + if (alreadyQueued) return + } + publishExecutor.execute { publishQueuedState() } + } + + private fun publishQueuedState() { + val json: String + synchronized(lock) { + json = queuedState ?: return + queuedState = null } try { val request = PutDataMapRequest.create(PATH_STATE) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt index 54f31c42..6c98b9b0 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -30,5 +30,23 @@ class PreviewBitmapUtil private constructor() { val colors = PreviewPalette.expand(bytes, 0, width * height, brightenGamma, green) return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) } + + /** + * A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes + * alpha; black, which the glasses show as unlit, is transparent), for + * showing glasses artwork on the phone's dark action bar. + */ + @JvmStatic + fun whiteIconFromGray(gray: ByteBuffer?, width: Int, height: Int): Bitmap { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw IllegalArgumentException("invalid gray icon buffer") + } + val bytes = ByteArray(width * height) + gray.get(bytes) + val colors = IntArray(width * height) { i -> + ((bytes[i].toInt() and 0xff) shl 24) or 0x00ffffff + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) + } } } diff --git a/CHANGELOG b/CHANGELOG index 83780177..c88a73aa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,11 @@ * Add config settings to disable animation categories * EvenHub compatibility uses menu animations and is closer to style matched +* Reduce phone CPU usage +* Fix a crash on install of larger EvenHub apps +* Fix line wrapping on EvenHub permissions dialog +* Putting on the glasses auto-wakes the screen +* Add a button to the phone screen top bar to open EvenHub app phone UI ## 0.8.0 - 2026-09-26 diff --git a/app/app.css b/app/app.css index 49c6dbee..f493f6d4 100644 --- a/app/app.css +++ b/app/app.css @@ -16,8 +16,9 @@ ActionItem { color: #ffffff; } -/* The main page's custom titleView: title left, connection status and the - * warning triangle right-aligned beside the overflow menu. */ +/* The main page's custom titleView: title left; the EvenHub app icon, + * connection status and the warning triangle right-aligned beside the + * overflow menu. */ .action-bar-title { color: #ffffff; font-size: 18; @@ -32,6 +33,16 @@ ActionItem { vertical-align: center; } +/* The foreground EvenHub app's icon (opens its phone UI); the padding widens + * the tap target. */ +.action-bar-app-icon { + width: 32; + height: 32; + padding: 5.5; + margin-right: 8; + vertical-align: center; +} + /* The emoji brings its own yellow; the padding widens the tap target. */ .action-bar-warning { font-size: 18; diff --git a/app/apps/evenhub/installed-apps.ts b/app/apps/evenhub/installed-apps.ts index ea05c4f1..a785f34f 100644 --- a/app/apps/evenhub/installed-apps.ts +++ b/app/apps/evenhub/installed-apps.ts @@ -51,9 +51,19 @@ export function installedEvenHubPackageId(appId: string): string | null { } } +// The parsed index for the last stored value: the launcher and the watch +// state both list installed apps on every refresh, and the sort's +// localeCompare dominated the cost. +let parsedIndex: { raw: string; apps: InstalledEvenHubApp[] } | null = null; + export function getInstalledEvenHubApps(): InstalledEvenHubApp[] { const raw = getStringSetting(STORAGE_KEY, ""); if (!raw) return []; + if (parsedIndex?.raw !== raw) parsedIndex = { raw, apps: parseInstalledIndex(raw) }; + return parsedIndex.apps.map((app) => ({ ...app })); +} + +function parseInstalledIndex(raw: string): InstalledEvenHubApp[] { try { const parsed = JSON.parse(raw); if (!Array.isArray(parsed)) return []; diff --git a/app/apps/evenhub/manager.ts b/app/apps/evenhub/manager.ts index 7b3b327c..0b12dabe 100644 --- a/app/apps/evenhub/manager.ts +++ b/app/apps/evenhub/manager.ts @@ -6,11 +6,12 @@ * * Glasses-first: launching does NOT take over the phone screen. The phone * stays on the Faceclaw dashboard; an app's phone UI is shown on demand - * (window menu -> "Show phone UI"), overlaying the dashboard, and the Back - * button returns to it. Apps are kept alive until explicitly closed (no memory - * eviction yet). + * (window menu -> "Show phone UI", or the app-icon button in the phone's + * action bar while the app is foreground on the glasses), overlaying the + * dashboard, and the Back button or the overlay's close button returns to it. + * Apps are kept alive until explicitly closed (no memory eviction yet). */ -import { Application } from "@nativescript/core"; +import { Application, type ImageSource } from "@nativescript/core"; import { type AppContext } from "../app-definition"; import { appFilesDirPath, @@ -23,6 +24,8 @@ import { EvenHubSession } from "./session"; import { createEvenHubWindow } from "./evenhub-window"; import { createEvenHubWebView, type EvenHubWebView } from "./webview"; import { shell } from "../../ui/shell/shell"; +import { GrayImage } from "../../graphics/image"; +import { grayImageToWhiteIconSource } from "../../native/gray-image-preview"; import { readEvenHubPackageManifest, installedEvenHubAppId, @@ -36,12 +39,31 @@ type RunningApp = { packageId: string; session: EvenHubSession; webView: EvenHubWebView; + /** The app's icon for the phone action bar, rendered on first use. */ + phoneIcon?: ImageSource | null; }; +/** The phone action bar's button for opening the foreground app's phone UI. */ +export type PhoneUiButton = { windowId: string; title: string; icon: ImageSource | null }; + +/** Pixel size the action-bar icon is rendered at (~21dp at xxhdpi). */ +const PHONE_ICON_SIZE = 63; + const running = new Map(); let nextSerial = 1; let phoneShownWindowId: string | null = null; let backHandlerRegistered = false; +const phoneShownListeners = new Set<() => void>(); + +/** Called whenever an app's phone UI is shown or hidden. Returns an unsubscribe. */ +export function onPhoneShownChanged(listener: () => void): () => void { + phoneShownListeners.add(listener); + return () => phoneShownListeners.delete(listener); +} + +function notifyPhoneShownChanged(): void { + for (const listener of phoneShownListeners) listener(); +} /** Intercept Android Back while an app's phone UI is overlaying the dashboard. */ function ensureBackHandler(): void { @@ -112,6 +134,7 @@ async function startApp( if (phoneShownWindowId === windowId) { webView.hideOnPhone(); phoneShownWindowId = null; + notifyPhoneShownChanged(); } webView.destroy(); cleanup?.(); @@ -205,15 +228,35 @@ export function showOnPhone(windowId: string): void { const app = running.get(windowId); if (!app) return; if (phoneShownWindowId && phoneShownWindowId !== windowId) hidePhone(); - app.webView.showOnPhone(); + app.webView.showOnPhone(hidePhone); phoneShownWindowId = windowId; + notifyPhoneShownChanged(); } -/** Return the phone to the dashboard (Back, or the shown app closing). */ +/** Return the phone to the dashboard (Back, the close button, or the shown app closing). */ export function hidePhone(): void { if (!phoneShownWindowId) return; running.get(phoneShownWindowId)?.webView.hideOnPhone(); phoneShownWindowId = null; + notifyPhoneShownChanged(); +} + +/** + * The foreground glasses window, when it is an EvenHub app whose phone UI + * isn't showing: the phone action bar then offers a button to show it. + */ +export function phoneUiButton(): PhoneUiButton | null { + const window = shell.foregroundWindow(); + const app = window ? running.get(window.windowId) : undefined; + if (!window || !app || phoneShownWindowId === app.windowId) return null; + if (app.phoneIcon === undefined) { + // The same artwork as the glasses sidebar (installed icon, else the + // generic package icon), drawn white. + const image = new GrayImage(PHONE_ICON_SIZE, PHONE_ICON_SIZE, 0); + window.drawIcon(image, 0, 0, PHONE_ICON_SIZE); + app.phoneIcon = grayImageToWhiteIconSource(image); + } + return { windowId: app.windowId, title: window.title, icon: app.phoneIcon }; } export function runningAppCount(): number { diff --git a/app/apps/evenhub/permission-dialog.ts b/app/apps/evenhub/permission-dialog.ts index 07977024..3f0560ff 100644 --- a/app/apps/evenhub/permission-dialog.ts +++ b/app/apps/evenhub/permission-dialog.ts @@ -1,6 +1,6 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { GrayImage } from "../../graphics/image"; -import { truncateText } from "../../graphics/textwrap"; +import { truncateText, wrapText } from "../../graphics/textwrap"; import { type InputEvent } from "../../ui/gestures"; import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers"; import { drawListScrollbar, drawSelectionHighlight } from "../../ui/menu"; @@ -8,9 +8,9 @@ import { permissionDetail, permissionLabel, type EvenHubPermission } from "./per import { openPrivacyPolicyOnPhone } from "./privacy-policy"; import { lineStep, listRowHeight } from "../../ui/metrics"; -const DIALOG_X = 8; -const DIALOG_Y = 8; -const DIALOG_WIDTH = 272; +/** Minimum gap between the dialog and the edges of the window it's centered in. */ +const DIALOG_MARGIN = 8; +const DIALOG_WIDTH = 320; const PADDING = 10; const PERM_GAP = 4; @@ -27,6 +27,8 @@ const PERM_GAP = 4; * list on screen. */ export class EvenHubPermissionDialogLayer implements Layer { + /** Floats in front of the window content, like the context menus. */ + readonly depth = 2; /** Starts on Allow; the optional privacy-policy action is inserted after it. */ private selectedIndex = 0; /** First content row drawn; rows above it are scrolled off. */ @@ -50,10 +52,12 @@ export class EvenHubPermissionDialogLayer implements Layer { paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { const font = getDefaultSmallFont(); - const { height: viewportHeight } = ctx.stack.getBaseSize(); + const { width: viewportWidth, height: viewportHeight } = ctx.stack.getBaseSize(); const image = paintBelow(); - const textWidth = DIALOG_WIDTH - 2 * PADDING - 4; - const left = DIALOG_X + PADDING + 2; + const dialogWidth = Math.min(DIALOG_WIDTH, viewportWidth - 2 * DIALOG_MARGIN); + const dialogX = ((viewportWidth - dialogWidth) / 2) | 0; + const textWidth = dialogWidth - 2 * PADDING - 4; + const left = dialogX + PADDING + 2; const headerStep = lineStep(font) + 2; const permLabelStep = lineStep(font) + 1; @@ -80,15 +84,17 @@ export class EvenHubPermissionDialogLayer implements Layer { for (let index = 0; index < this.permissions.length; index++) { const permission = this.permissions[index]!; const detail = permissionDetail(permission); + // One row per wrapped line, so the explanation scrolls line by line. + const detailLines = detail ? wrapText(font, detail, textWidth - 8) : []; const gapBelow = PERM_GAP + (index === this.permissions.length - 1 ? 2 : 0); rows.push({ - height: permLabelStep + (detail ? 0 : gapBelow), + height: permLabelStep + (detailLines.length ? 0 : gapBelow), draw: (top) => image.drawText(font, left, top, truncateText(font, permissionLabel(permission.name), textWidth), 220), }); - if (detail) { + for (let line = 0; line < detailLines.length; line++) { rows.push({ - height: permDetailStep + gapBelow, - draw: (top) => image.drawText(font, left + 8, top, truncateText(font, detail, textWidth - 8), 140), + height: permDetailStep + (line === detailLines.length - 1 ? gapBelow : 0), + draw: (top) => image.drawText(font, left + 8, top, detailLines[line]!, 140), }); } } @@ -98,15 +104,16 @@ export class EvenHubPermissionDialogLayer implements Layer { draw: (top) => { const selected = this.atBottom() && index === this.selectedIndex; if (selected) { - drawSelectionHighlight(image, DIALOG_X + 12, top, DIALOG_WIDTH - 24, actionRowH - 1, focused, 8); + drawSelectionHighlight(image, dialogX + 12, top, dialogWidth - 24, actionRowH - 1, focused, 8); } - image.drawText(font, DIALOG_X + 22, top + 3, actions[index]!, selected ? 255 : 200); + image.drawText(font, dialogX + 22, top + 3, actions[index]!, selected ? 255 : 200); }, }); } const contentHeight = rows.reduce((sum, row) => sum + row.height, 0); - const height = Math.min(contentHeight + 2 * PADDING, viewportHeight - 2 * DIALOG_Y); + const height = Math.min(contentHeight + 2 * PADDING, viewportHeight - 2 * DIALOG_MARGIN); + const dialogY = ((viewportHeight - height) / 2) | 0; const visibleHeight = height - 2 * PADDING; // The bottommost scroll position: the largest suffix of rows that fits. @@ -120,13 +127,13 @@ export class EvenHubPermissionDialogLayer implements Layer { this.firstRow = Math.max(0, Math.min(this.firstRow, maxFirstRow)); // Fill 1 (transparent color key is 0), outline for the dialog edge. - image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 1); - image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 72); + image.fillRoundedRect(dialogX, dialogY, dialogWidth, height, 1); + image.drawRoundedRect(dialogX, dialogY, dialogWidth, height, 72); - let y = DIALOG_Y + PADDING; + let y = dialogY + PADDING; for (let index = this.firstRow; index < rows.length; index++) { const row = rows[index]!; - if (y + row.height > DIALOG_Y + PADDING + visibleHeight) break; + if (y + row.height > dialogY + PADDING + visibleHeight) break; row.draw(y); y += row.height; } @@ -134,8 +141,8 @@ export class EvenHubPermissionDialogLayer implements Layer { if (maxFirstRow > 0) { drawListScrollbar( image, - DIALOG_X + DIALOG_WIDTH - 7, - DIALOG_Y + PADDING, + dialogX + dialogWidth - 7, + dialogY + PADDING, visibleHeight, this.firstRow, rows.length - maxFirstRow, diff --git a/app/apps/evenhub/webview.ts b/app/apps/evenhub/webview.ts index 06c0f0d3..655822ac 100644 --- a/app/apps/evenhub/webview.ts +++ b/app/apps/evenhub/webview.ts @@ -22,11 +22,13 @@ import { FACECLAW_VERSION } from "../../version"; declare const android: any; declare const com: any; +declare const java: any; export type EvenHubWebView = { evaluateJs: (js: string) => void; destroy: () => void; - showOnPhone: () => void; + /** Show the app's phone UI; `onClose` runs when its top bar's close button is tapped. */ + showOnPhone: (onClose: () => void) => void; hideOnPhone: () => void; /** The native android.webkit.WebView, for the host. */ native: unknown; @@ -118,8 +120,13 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { console.warn(`evenhub webview destroy failed: ${error}`); } }, - showOnPhone: () => - nativeHost.showOnPhone(Application.android?.foregroundActivity ?? Application.android?.startActivity ?? null, webView), + showOnPhone: (onClose: () => void) => + nativeHost.showOnPhone( + Application.android?.foregroundActivity ?? Application.android?.startActivity ?? null, + webView, + session.manifest.name, + new java.lang.Runnable({ run: onClose }), + ), hideOnPhone: () => nativeHost.hideOnPhone(), }; } diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 1b5b0978..e74a24b0 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -78,7 +78,14 @@ import { installedEvenHubPackageId, uninstallEvenHubPackage, } from "../apps/evenhub/installed-apps"; -import { closeRunningPackage, launchInstalledPackage } from "../apps/evenhub/manager"; +import { + closeRunningPackage, + launchInstalledPackage, + onPhoneShownChanged, + phoneUiButton, + showOnPhone as showEvenHubPhoneUi, + type PhoneUiButton, +} from "../apps/evenhub/manager"; import { openEvenHubStoreForPackage } from "../apps/evenhub"; import { isInstalledPackagePresent } from "../apps/evenhub/updates"; import { wearerVerificationOptions } from "../apps/microphones/speakers"; @@ -133,6 +140,11 @@ export type DashboardSnapshot = { * nothing is paired, so "Disconnected" would be the wrong label. */ previewMode: boolean; + /** + * The foreground glasses window's EvenHub phone UI, when it can be opened + * (the action bar's app-icon button); null otherwise. + */ + evenHubPhoneUi: PhoneUiButton | null; }; type DashboardListener = (snapshot: DashboardSnapshot) => void; @@ -149,6 +161,9 @@ const SHELL_REFRESH_INTERVAL_MS = 60_000; // schedules a refresh), so this just covers anything that slips through. const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; +// The lock screen is a short notice, not something to read at leisure; it +// ignores the screen-timeout setting (including "never"). +const LOCK_SCREEN_TIMEOUT_MS = 15_000; const EVENHUB_SCREEN_OFF_SUSPEND_DELAY_MS = 5_000; const EVENHUB_WAKE_READY_TIMEOUT_MS = 4_500; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; @@ -358,6 +373,8 @@ class DashboardController { launchApp: (appId) => this.launchApp(appId), requestShellRender: () => this.requestShellRender(), }); + // The action bar's EvenHub phone-UI button hides while that UI is showing. + onPhoneShownChanged(() => this.emit()); shell.configure({ actions: { ...sharedActions, @@ -365,7 +382,8 @@ class DashboardController { // surface, so their repaints go through the shell render path. requestRender: () => this.requestShellRender(), }, - getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()), + getScreenTimeoutMs: () => + this.glassesLocked ? LOCK_SCREEN_TIMEOUT_MS : screenTimeoutSettingToMs(screenTimeoutSetting.get()), requestShellRender: () => this.requestShellRender(), prepareVoiceCapture: () => this.prepareVoiceCapture(), onKeyboardInputChanged: (session) => { @@ -626,6 +644,9 @@ class DashboardController { } private handleWearState(wearing: boolean): void { + // Only an observed OFF_HEAD -> ON_HEAD transition counts as putting the + // glasses on; the first snapshot of a session (null) is just the status. + const putOn = wearing && this.glassesWorn === false; this.glassesWorn = wearing; updateGlassesPresence({ worn: wearing }); this.appendLog(wearing ? "glasses wear state: ON_HEAD" : "glasses wear state: OFF_HEAD"); @@ -633,6 +654,19 @@ class DashboardController { if (!wearing && this.phoneLocked && lockScreenEnabledSetting.get()) { this.setGlassesLocked(true, "glasses removed while phone locked"); } + if (putOn) this.wakeForGlassesPutOn(); + } + + /** + * Putting the glasses on wakes the screen, like a double-tap would. While + * locked that shows the lock-screen notice (which then times out on its + * own short timer); otherwise it lands on the sidebar as a normal wake. + */ + private wakeForGlassesPutOn(): void { + if (this.phase !== "connected" || !this.communicator || shell.isScreenOn()) return; + // wake() -> onScreenStateChanged(true) dismisses a showing Glanceboard, + // runs the EvenHub wake barrier and repaints the shell. + if (shell.wake("sidebar")) this.appendLog("screen woken: glasses put on"); } private handlePhoneLockState(locked: boolean): void { @@ -1005,9 +1039,15 @@ class DashboardController { fontsMissingWarningVisible: this.fontsMissingWarningVisible, alarmReliabilityMessage: this.alarmReliabilityMessage, previewMode: this.isPreviewDisplayActive(), + evenHubPhoneUi: phoneUiButton(), }; } + /** Overlay an EvenHub app's phone UI (the action bar's app-icon button). */ + showEvenHubPhoneUi(windowId: string): void { + showEvenHubPhoneUi(windowId); + } + /** * Message to show in place of the display preview, or "" to show the preview. * @@ -2475,6 +2515,13 @@ class DashboardController { return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); + // An identical repaint (e.g. every in-process window re-renders when the + // switcher brings it to the foreground) would flatten, encode and + // composite only to be deduped before sending. + if (display.isSurfaceCurrent(surfaceId, fingerprint)) { + frameTimings.finishFrame(frameId, "discarded: surface unchanged"); + return; + } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)); diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 7e8ec2dd..6cb89bf3 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -362,9 +362,13 @@ export class IosPreviewController { // Sample false to catch a notification missed during suspension. Do not // sample true here: WillBecomeUnavailable precedes the property transition. if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true) + // Putting the glasses on (an observed OFF_HEAD -> ON_HEAD transition, not + // a session's first snapshot) wakes the screen, to the lock notice if locked. + const putOn = wearing && this.glassesWorn === false this.glassesWorn = wearing this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`) if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true) + if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar') } private setGlassesLocked(locked: boolean): void { if (locked === this.glassesLocked) return diff --git a/app/graphics/display-list.ts b/app/graphics/display-list.ts index 00109121..924f87d7 100644 --- a/app/graphics/display-list.ts +++ b/app/graphics/display-list.ts @@ -36,14 +36,34 @@ export type DisplayList = { }; export type PlacedDisplayList = { displayList: DisplayList; x: number; y: number; width: number; height: number; depth: number }; +/** + * Growable byte buffer. Lists are encoded on every frame that carries one, and + * twice more for its fingerprint and the preview; resources (whole menu rows, + * scroll strips) are tens of KB, so they go in with one set() rather than a + * push per byte. + */ class Writer { - bytes: number[] = []; - u8(n: number): void { this.bytes.push(integer(n, 0, 255)); } + private buffer = new Uint8Array(256); + length = 0; + private reserve(count: number): void { + const needed = this.length + count; + if (needed <= this.buffer.length) return; + let size = this.buffer.length * 2; + while (size < needed) size *= 2; + const grown = new Uint8Array(size); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + u8(n: number): void { integer(n, 0, 255); this.reserve(1); this.buffer[this.length++] = n; } i8(n: number): void { this.u8(integer(n, -128, 127) & 255); } - u16(n: number): void { integer(n, 0, 65535); this.bytes.push(n & 255, n >>> 8); } + u16(n: number): void { integer(n, 0, 65535); this.reserve(2); this.buffer[this.length++] = n & 255; this.buffer[this.length++] = n >>> 8; } i16(n: number): void { this.u16(integer(n, -32768, 32767) & 65535); } u32(n: number): void { integer(n, 0, 4294967295); this.u16(n & 65535); this.u16(n >>> 16); } - raw(bytes: ArrayLike): void { for (let i = 0; i < bytes.length; i++) this.bytes.push(bytes[i]); } + raw(bytes: ArrayLike): void { this.reserve(bytes.length); this.buffer.set(bytes, this.length); this.length += bytes.length; } + /** The written bytes, in a buffer of their own. */ + bytes(): Uint8Array { return this.buffer.slice(0, this.length); } + /** A view of the written bytes; valid until the next write. */ + view(): Uint8Array { return this.buffer.subarray(0, this.length); } } /** Length-delimited bridge envelope; call operands use fixed widths except extended x/y. */ @@ -85,9 +105,9 @@ export function encodeDisplayList(placed: PlacedDisplayList, now = Date.now()): } } } - if (out.bytes.length > 4 * 1024 * 1024) throw new Error('Display list exceeds bridge limit'); - const header = new Writer(); header.u8(DISPLAY_LIST_RECORD); header.u32(out.bytes.length); header.raw(out.bytes); - return Uint8Array.from(header.bytes); + if (out.length > 4 * 1024 * 1024) throw new Error('Display list exceeds bridge limit'); + const header = new Writer(); header.u8(DISPLAY_LIST_RECORD); header.u32(out.length); header.raw(out.view()); + return header.bytes(); } /** Replay-record sizes by tag, as in glyph-wire's frame draw buffer. */ diff --git a/app/graphics/image.ts b/app/graphics/image.ts index c841071b..6f3164c8 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -277,17 +277,28 @@ export class GrayImage { const destX = dx | 0; const destY = dy | 0; - for (let row = 0; row < copyHeight; row++) { - const sy = srcY + row; - const ty = destY + row; - if (sy < 0 || sy >= source.height || ty < 0 || ty >= this.height) continue; - for (let col = 0; col < copyWidth; col++) { - const sx = srcX + col; - const tx = destX + col; - if (sx < 0 || sx >= source.width || tx < 0 || tx >= this.width) continue; - const value = source.pixels[sy * source.width + sx] ?? 0; + // Clip once, then copy row spans: flattening a frame blits every plane, + // and per-pixel bounds checks made that a sizeable share of a paint. + const colStart = Math.max(0, -destX); + const colEnd = Math.min(copyWidth, source.width - srcX, this.width - destX); + const rowStart = Math.max(0, -destY); + const rowEnd = Math.min(copyHeight, source.height - srcY, this.height - destY); + const span = colEnd - colStart; + const src = source.pixels; + const dst = this.pixels; + // A blit within one image copies pixel by pixel, forwards, as it always has. + const bulk = !opts.transparentZero && source !== this; + for (let row = rowStart; span > 0 && row < rowEnd; row++) { + const from = (srcY + row) * source.width + srcX + colStart; + const to = (destY + row) * this.width + destX + colStart; + if (bulk) { + dst.set(src.subarray(from, from + span), to); + continue; + } + for (let i = 0; i < span; i++) { + const value = src[from + i]!; if (opts.transparentZero && value === 0) continue; - this.pixels[ty * this.width + tx] = value; + dst[to + i] = value; } } @@ -398,7 +409,7 @@ export class GrayImage { hash = mixInt(hash, placed.source.height); hash = mixInt(hash, placed.source.sourceContentHash32()); if (placed.presentation) { const p = placed.presentation; for (const value of [p.radius, p.background, p.border, p.depth, p.mode === "image" ? 1 : p.mode === "masked-image" ? 2 : 0]) hash = mixInt(hash, value); - if (p.displayList) for (const byte of encodeDisplayList({ displayList: p.displayList, x: 0, y: 0, width: placed.source.width, height: placed.source.height, depth: p.depth }, p.displayList.timeline?.startedAt ?? 0)) hash = mixInt(hash, byte); + if (p.displayList) hash = mixInt(hash, bytesHash32(encodeDisplayList({ displayList: p.displayList, x: 0, y: 0, width: placed.source.width, height: placed.source.height, depth: p.depth }, p.displayList.timeline?.startedAt ?? 0))); } } else { hash = mixInt(hash, 0xf17e); @@ -415,28 +426,9 @@ export class GrayImage { return hash >>> 0; } - /** - * FNV-1a over the raster pixels. Four bytes at a time where alignment - * allows: this runs on every frame's fingerprint, over the whole screen, on - * the thread that is about to hand the frame to the transport. - */ + /** Hash of the raster pixels; see bytesHash32. */ private pixelsHash32(): number { - const pixels = this.pixels; - let hash = 2166136261; - let i = 0; - if ((pixels.byteOffset & 3) === 0) { - const words = new Uint32Array(pixels.buffer, pixels.byteOffset, pixels.length >>> 2); - for (let w = 0; w < words.length; w++) { - hash ^= words[w]!; - hash = Math.imul(hash, 16777619); - } - i = words.length << 2; - } - for (; i < pixels.length; i++) { - hash ^= pixels[i]!; - hash = Math.imul(hash, 16777619); - } - return hash >>> 0; + return bytesHash32(this.pixels); } /** @@ -744,6 +736,29 @@ function clampByte(value: number): number { } /** Fold a 32-bit int into an FNV-style running hash. */ +/** + * FNV-1a over bytes. Four bytes at a time where alignment allows: this runs on + * every frame's fingerprint, over the whole screen, on the thread that is + * about to hand the frame to the transport. + */ +function bytesHash32(bytes: Uint8Array): number { + let hash = 2166136261; + let i = 0; + if ((bytes.byteOffset & 3) === 0) { + const words = new Uint32Array(bytes.buffer, bytes.byteOffset, bytes.length >>> 2); + for (let w = 0; w < words.length; w++) { + hash ^= words[w]!; + hash = Math.imul(hash, 16777619); + } + i = words.length << 2; + } + for (; i < bytes.length; i++) { + hash ^= bytes[i]!; + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} + function mixInt(hash: number, value: number): number { hash ^= value | 0; return Math.imul(hash, 16777619); diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index 47fe1231..704a4a66 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -536,6 +536,14 @@ export class FaceclawCommunicatorBridge { await this.enqueueNativeCall(() => { this.communicator.setScreenBlankedBlanked(Boolean(blanked)); }); } + /** + * See the Android bridge. Not wired to the shared compositor's check on iOS + * yet, so every repaint is submitted as before. + */ + isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean { + return false; + } + async submitSurfaceFrame( surfaceId: string, pixels8bpp: Uint8Array, diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 37b5f479..ced591d5 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -133,6 +133,8 @@ export class FaceclawCommunicatorBridge { private javaCallQueue: Promise = Promise.resolve(); /** Calls waiting on javaCallQueue; 0 means enqueueJavaCall's fast path is safe. */ private queuedJavaCalls = 0; + /** Queued calls that may change what a surface holds; see isSurfaceCurrent. */ + private queuedSurfaceChanges = 0; // Reused Java-side buffers for byte payloads; passing a JS ArrayBuffer to // Java leaks it (see java-direct-buffer.ts). Loaded inside the queued call, // immediately before the synchronous Java call that consumes them. @@ -267,8 +269,12 @@ export class FaceclawCommunicatorBridge { * the frame timings), which is pure latency on a call the caller is already * awaiting. Off by default: a call that might block for a while should keep * yielding to the main looper first. + * + * preservesSurfaces marks a call that neither changes a surface's retained + * content nor leaves a change waiting for a later composite (it composites + * itself), so having it queued does not make isSurfaceCurrent answer false. */ - private enqueueJavaCall(operation: () => T, inlineWhenIdle = false): Promise { + private enqueueJavaCall(operation: () => T, inlineWhenIdle = false, preservesSurfaces = false): Promise { if (inlineWhenIdle && this.queuedJavaCalls === 0) { try { return Promise.resolve(operation()); @@ -277,10 +283,12 @@ export class FaceclawCommunicatorBridge { } } this.queuedJavaCalls++; + if (!preservesSurfaces) this.queuedSurfaceChanges++; const run = () => new Promise((resolve, reject) => { setTimeout(() => { this.queuedJavaCalls--; + if (!preservesSurfaces) this.queuedSurfaceChanges--; try { resolve(operation()); } catch (error) { @@ -549,6 +557,7 @@ export class FaceclawCommunicatorBridge { await this.enqueueJavaCall( () => this.communicator.submitShellScene(this.shellSceneBuffer.load(snapshot), paintMs, frameId), true, + true, ); } @@ -562,7 +571,19 @@ export class FaceclawCommunicatorBridge { async setSurfaceVisible(id: string, visible: boolean): Promise { await this.enqueueJavaCall(() => { this.communicator.setSurfaceVisible(id, Boolean(visible)); - }); + }, false, true); + } + + /** + * Whether submitting a full-surface frame with this fingerprint would + * change nothing: Java already holds that content for the surface, and no + * queued or not-yet-composited change is waiting on a composite. Lets a + * repaint that came out identical (a window re-rendered when it comes to + * the foreground, say) skip flattening and submitting its frame. + */ + isSurfaceCurrent(surfaceId: string, fingerprint: string): boolean { + if (this.queuedSurfaceChanges > 0) return false; + return Boolean(this.communicator.isSurfaceCurrent(surfaceId, fingerprint)); } /** diff --git a/app/native/gray-image-preview.ts b/app/native/gray-image-preview.ts index fc8b1656..e59c2655 100644 --- a/app/native/gray-image-preview.ts +++ b/app/native/gray-image-preview.ts @@ -27,3 +27,21 @@ export function grayImageToPreviewSource(image: GrayImage): ImageSource | null { ); return new ImageSource(bitmap); } + +/** + * A glasses icon as a white-on-transparent phone image (gray becomes alpha), + * for the dark action bar. + */ +export function grayImageToWhiteIconSource(image: GrayImage): ImageSource | null { + if (!global.isAndroid) { + return null; + } + + const baked = image.withDrawsBaked(); + const bitmap = com.faceclaw.app.PreviewBitmapUtil.whiteIconFromGray( + copyToJavaByteBuffer(baked.pixels), + baked.width, + baked.height, + ); + return new ImageSource(bitmap); +} diff --git a/app/native/phone-battery.ts b/app/native/phone-battery.ts index b758fcd3..23bd5e9f 100644 --- a/app/native/phone-battery.ts +++ b/app/native/phone-battery.ts @@ -7,7 +7,25 @@ export type PhoneBatteryState = { charging: boolean | null; }; +// The Android read is a sticky-broadcast binder round trip, and the top bar +// asks on every shell paint; the level moves far slower than that. +const ANDROID_CACHE_MS = 5000; +let cachedState: PhoneBatteryState | null = null; +let cachedAtMs = 0; + export function readPhoneBatteryState(): PhoneBatteryState { + if (global.isAndroid) { + const now = Date.now(); + if (cachedState === null || now - cachedAtMs >= ANDROID_CACHE_MS || now < cachedAtMs) { + cachedState = readPhoneBatteryStateUncached(); + cachedAtMs = now; + } + return { ...cachedState }; + } + return readPhoneBatteryStateUncached(); +} + +function readPhoneBatteryStateUncached(): PhoneBatteryState { if (global.isIOS) { const device = UIDevice.currentDevice const level = device.batteryLevel diff --git a/app/native/preview-display.ios.ts b/app/native/preview-display.ios.ts index 2a70a769..655ed620 100644 --- a/app/native/preview-display.ios.ts +++ b/app/native/preview-display.ios.ts @@ -18,6 +18,7 @@ export type DisplayTarget = Pick< | "setUnderlayDim" | "setScreenBlanked" | "submitSurfaceFrame" + | "isSurfaceCurrent" | "submitShellScene" | "waitForFrameFinished" | "getCompositePreview" @@ -109,6 +110,11 @@ export class PreviewDisplayTarget implements DisplayTarget { this.composited(); } + /** Never skips: preview frames are cheap and nothing is transmitted. */ + isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean { + return false; + } + async submitShellScene(bytes: Uint8Array, _paintMs = 0, _frameId = 0): Promise { this.compositor.setShellScene(bytes); this.composited(); diff --git a/app/native/preview-display.ts b/app/native/preview-display.ts index 53e3e927..4f6d7531 100644 --- a/app/native/preview-display.ts +++ b/app/native/preview-display.ts @@ -24,6 +24,7 @@ export type DisplayTarget = Pick< | "setUnderlayDim" | "setScreenBlanked" | "submitSurfaceFrame" + | "isSurfaceCurrent" | "submitShellScene" | "waitForFrameFinished" | "getCompositePreview" @@ -109,6 +110,11 @@ export class PreviewDisplayTarget implements DisplayTarget { this.compositor.setSurfaceDepth(id, Math.round(depth)); } + /** Never skips: preview frames are cheap and nothing is transmitted. */ + isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean { + return false; + } + async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise { this.compositor.submitShellScene(this.shellSceneBuffer.load(bytes), paintMs, frameId); } diff --git a/app/native/settings-store.ts b/app/native/settings-store.ts index 3f3f43da..62992be2 100644 --- a/app/native/settings-store.ts +++ b/app/native/settings-store.ts @@ -8,17 +8,35 @@ import { Utils } from "@nativescript/core"; declare const com: any; +declare const android: any; let javaInstance: any = null; // The Java-side listener proxy must stay referenced or it gets GC'd. let retainedListenerProxy: any = null; const changeListeners = new Set<(key: string) => void>(); +// Values already read through JNI, which cost a bridge call each and are read +// many times per paint (display mode, fonts, clock format). Only used while +// this isolate's Java listener is registered on a thread with a Looper: every +// write, from any isolate, notifies that listener, which drops the key before +// any other change listener runs. A write from another isolate is visible here +// once its notification has been dispatched (one message-loop tick), the same +// point at which change listeners hear about it. +// Strings are cached without a default (null = unset). Booleans are cached per +// default, since the Java getter cannot report "unset", except after a write +// from this isolate (defaultValue null: stored, so any default reads it). +const stringCache = new Map(); +const booleanCache = new Map(); +let cachingEnabled: boolean | null = null; + /** Release this isolate's native observer before its worker is terminated. */ export function disposeSettingsStore(): void { changeListeners.clear(); if (retainedListenerProxy !== null) getJava().unregisterListener(retainedListenerProxy); retainedListenerProxy = null; + cachingEnabled = false; + stringCache.clear(); + booleanCache.clear(); } function getJava(): any { @@ -31,20 +49,80 @@ function getJava(): any { return javaInstance; } +function canCache(): boolean { + if (cachingEnabled === null) { + cachingEnabled = false; + try { + // FaceclawSettings dispatches notifications through the registering + // thread's Looper; without one this isolate would never hear of writes. + if (android.os.Looper.myLooper() !== null) { + ensureJavaListener(); + cachingEnabled = true; + } + } catch { + cachingEnabled = false; + } + } + return cachingEnabled; +} + export function getStringSetting(key: string, defaultValue: string): string { - return String(getJava().getString(key, defaultValue)); + if (!canCache()) return String(getJava().getString(key, defaultValue)); + let value = stringCache.get(key); + if (value === undefined) { + const raw = getJava().getString(key, null); + value = raw === null || raw === undefined ? null : String(raw); + stringCache.set(key, value); + } + return value ?? defaultValue; } export function setStringSetting(key: string, value: string): void { getJava().setString(key, value); + if (cachingEnabled) stringCache.set(key, value === null || value === undefined ? null : String(value)); } export function getBooleanSetting(key: string, defaultValue: boolean): boolean { - return Boolean(getJava().getBoolean(key, defaultValue)); + if (!canCache()) return Boolean(getJava().getBoolean(key, defaultValue)); + const cached = booleanCache.get(key); + if (cached !== undefined && (cached.defaultValue === null || cached.defaultValue === defaultValue)) { + return cached.value; + } + const value = Boolean(getJava().getBoolean(key, defaultValue)); + booleanCache.set(key, { defaultValue, value }); + return value; } export function setBooleanSetting(key: string, value: boolean): void { getJava().setBoolean(key, value); + if (cachingEnabled) booleanCache.set(key, { defaultValue: null, value: Boolean(value) }); +} + +function forgetCachedSetting(key: string | null | undefined): void { + if (key === null || key === undefined) { + stringCache.clear(); + booleanCache.clear(); + return; + } + stringCache.delete(key); + booleanCache.delete(key); +} + +function ensureJavaListener(): void { + if (retainedListenerProxy !== null) return; + retainedListenerProxy = new com.faceclaw.app.FaceclawSettingsListener({ + onSettingChanged: (key: string) => { + forgetCachedSetting(key === null || key === undefined ? key : String(key)); + for (const registered of Array.from(changeListeners)) { + try { + registered(String(key)); + } catch (error) { + console.warn("settings change listener failed", error); + } + } + }, + }); + getJava().registerListener(retainedListenerProxy); } /** @@ -53,20 +131,7 @@ export function setBooleanSetting(key: string, value: boolean): void { * change. */ export function onSettingsStoreChanged(listener: (key: string) => void): () => void { - if (retainedListenerProxy === null) { - retainedListenerProxy = new com.faceclaw.app.FaceclawSettingsListener({ - onSettingChanged: (key: string) => { - for (const registered of Array.from(changeListeners)) { - try { - registered(String(key)); - } catch (error) { - console.warn("settings change listener failed", error); - } - } - }, - }); - getJava().registerListener(retainedListenerProxy); - } + ensureJavaListener(); changeListeners.add(listener); return () => { changeListeners.delete(listener); diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml index e80b711b..8b7d5cfb 100644 --- a/app/phone-ui/main-page.xml +++ b/app/phone-ui/main-page.xml @@ -1,11 +1,13 @@ - - + + diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts index 407a1948..04bdc9f3 100644 --- a/app/phone-ui/main-view-model.ts +++ b/app/phone-ui/main-view-model.ts @@ -25,6 +25,7 @@ import { isAutoReconnectSuppressed, resumeAutoReconnect } from "../g2/reconnect- import { isPreviewOnlyMode } from "./onboarding-state"; import { formatErrorMessage } from "../util/format-error"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image"; +import { type PhoneUiButton } from "../apps/evenhub/manager"; const LENS_ASPECT_RATIO = G2_LENS_WIDTH / G2_LENS_HEIGHT; @@ -63,6 +64,7 @@ export class MainViewModel extends RemoteControlsViewModel { private _alarmReliabilityMessage = ""; private _warningsModalVisible = false; private _previewMode = false; + private _evenHubPhoneUi: PhoneUiButton | null = null; private _phase: "disconnected" | "connecting" | "connected" | "charging" | "disconnecting" = "disconnected"; // A new view model is built on every navigation to the main page; these @@ -112,6 +114,7 @@ export class MainViewModel extends RemoteControlsViewModel { this.batteryOptimizationWarningVisible = snapshot.batteryOptimizationWarningVisible; this.fontsMissingWarningVisible = snapshot.fontsMissingWarningVisible; this.alarmReliabilityMessage = snapshot.alarmReliabilityMessage; + this.evenHubPhoneUi = snapshot.evenHubPhoneUi; this.refreshPadFocusLine(); })); // Brightness / display mode can change from the glasses' Settings app too. @@ -773,6 +776,28 @@ export class MainViewModel extends RemoteControlsViewModel { return this._warningsModalVisible ? "visible" : "collapse"; } + set evenHubPhoneUi(value: PhoneUiButton | null) { + const current = this._evenHubPhoneUi; + if (current?.windowId === value?.windowId && current?.icon === value?.icon) return; + this._evenHubPhoneUi = value; + this.notifyPropertyChange("evenHubPhoneUiIcon", this.evenHubPhoneUiIcon); + this.notifyPropertyChange("evenHubPhoneUiVisibility", this.evenHubPhoneUiVisibility); + } + + /** The foreground EvenHub app's icon, which opens its phone UI when tapped. */ + get evenHubPhoneUiIcon(): ImageSource | null { + return this._evenHubPhoneUi?.icon ?? null; + } + + get evenHubPhoneUiVisibility(): "visible" | "collapse" { + return this._evenHubPhoneUi?.icon ? "visible" : "collapse"; + } + + onEvenHubPhoneUiTap(): void { + const windowId = this._evenHubPhoneUi?.windowId; + if (windowId) dashboardController.showEvenHubPhoneUi(windowId); + } + onWarningIconTap(): void { this.setWarningsModalVisible(true); } diff --git a/app/ui/shell/assistant.ts b/app/ui/shell/assistant.ts index 1df039d7..2fe25fbc 100644 --- a/app/ui/shell/assistant.ts +++ b/app/ui/shell/assistant.ts @@ -3,13 +3,13 @@ import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../gestures"; import { Layer, type LayerActions, type LayerContext } from "../layers"; import { createInputDialogMenu, paintInputDialog, type InputDialogRow } from "./input-dialog"; -const FOLLOW_UP_ROW = 0; +const FOLLOW_UP_ROW = 1; const MENU_ROWS: readonly InputDialogRow[] = [ - { label: "Follow-up", dim: false }, { label: "Done", dim: false }, + { label: "Follow-up", dim: false }, ]; -/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */ +/** thinking: turn running; done/error: finished, showing the Done/Follow-up menu. */ type AssistantPhase = "thinking" | "done" | "error"; export type AssistantLayerCallbacks = { @@ -25,7 +25,7 @@ export type AssistantLayerCallbacks = { /** * The assistant overlay: streamed reply text with a status line for tool - * activity, and a Follow-up / Done menu once the turn ends. Double-click + * activity, and a Done / Follow-up menu once the turn ends. Double-click * cancels an in-flight turn. The shell owns the AssistantSession and drives * this layer's state through the on* methods as the turn streams. */ @@ -99,7 +99,7 @@ export class AssistantLayer implements Layer { } return; } - // done / error: the Follow-up / Done menu. + // done / error: the Done / Follow-up menu. switch (event.type) { case "scroll-up": case "scroll-down": diff --git a/app/ui/shell/open-apps-persistence.ts b/app/ui/shell/open-apps-persistence.ts index 2528bede..8f5cd4d6 100644 --- a/app/ui/shell/open-apps-persistence.ts +++ b/app/ui/shell/open-apps-persistence.ts @@ -14,8 +14,9 @@ export type PersistedOpenApps = { const STATE_VERSION = 1; const FILE_NAME = "open-apps.json"; -// Coalesces bursts (restore, sidebar scrolling) into one write; short enough -// that a build-reinstall kill right after a change rarely loses it. +// Coalesces bursts (restore, sidebar scrolling) into one write once they +// settle; short enough that a build-reinstall kill right after a change +// rarely loses it. const WRITE_DELAY_MS = 1000; function openAppsFilePath(): string { @@ -50,7 +51,10 @@ let pendingState: PersistedOpenApps | null = null; /** Save the open-app state (debounced; the latest state wins). */ export function savePersistedOpenApps(state: PersistedOpenApps): void { pendingState = state; - if (writeTimer !== null) return; + // Restarted on every change: scrolling through the switcher changes the + // foreground app a few times a second, and each synchronous write costs + // several milliseconds of main thread. + if (writeTimer !== null) clearTimeout(writeTimer); writeTimer = setTimeout(() => { writeTimer = null; const state = pendingState; diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 1a37a0e3..1db2d922 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -646,7 +646,7 @@ class Shell { // The keyboard dialog likewise: typing happens on the phone, not the // ring. An in-flight assistant turn suspends it for the same reason (a // tool loop can run for a while with no input); once the turn ends and - // the Follow-up/Done menu is showing, the normal idle timeout resumes. + // the Done/Follow-up menu is showing, the normal idle timeout resumes. if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) { this.lastInputAtMs = nowMs; return false; diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt index fb180979..fe9bf23b 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt @@ -63,7 +63,7 @@ class ScenePlanner(private val cache: ResourceCacheState) { val calls = ArrayList() for (top in 0 until height step 32) { val rows = minOf(32, height - top) - if (old != null && (top * stride until (top + rows) * stride).all { old[it] == next[it] }) continue + if (old != null && sameBytes(old, next, top * stride, (top + rows) * stride)) continue var left = width; var right = -1 for (y in top until top + rows) for (x in 0 until width) { val p = y * stride + x / 2; val shift = if (x % 2 == 0) 4 else 0 @@ -75,4 +75,12 @@ class ScenePlanner(private val cache: ResourceCacheState) { } return calls } + private fun sameBytes(a: ByteArray, b: ByteArray, from: Int, to: Int): Boolean { + var i = from + while (i < to) { + if (a[i] != b[i]) return false + i++ + } + return true + } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index c06cead3..4796d89c 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -1036,6 +1036,10 @@ class GlassesSessionCore( storeDesiredComposite(composite, paintMs, frameId) } + /** See [SurfaceCompositor.isSurfaceCurrent]. */ + fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean = + compositor.isSurfaceCurrent(surfaceId, fingerprint) + /** Store a composite as the desired frame unless a newer one won the race. */ internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, paintMs: Int, frameId: Int) { var supersededFrameId = 0 diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt index fda4e8e6..40328574 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt @@ -2,11 +2,14 @@ package com.faceclaw.app /** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int, - val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List = emptyList()) : RetainedDrawing { - val resource = CachedResource(DrawProtocol.rawImage(width, height, packed)) + val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List = emptyList(), + /** [packed] as a raw-image resource; translated copies share it (and its hash) instead of rebuilding it. */ + val resource: CachedResource = CachedResource(DrawProtocol.rawImage(width, height, packed))) : RetainedDrawing { override val resources = listOf(resource) - override val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}" - override fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask) + override val fingerprint: String by lazy { + "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}" + } + override fun translated(dx: Int, dy: Int) = if (dx == 0 && dy == 0) this else MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask, resource) override fun calls(ids: IntArray, nowMs: Long) = calls(ids.single()) fun calls(id: Int): List { val image = when (kind) { diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt index 58d3c3bf..57d92c5f 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt @@ -43,11 +43,19 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr return maxOf(1, (((value * dim) + 128) shr 8)) } + /** + * The rest of [reader] as a heap-array reader. Parsing reads field by field, and through a + * platform buffer (Android's direct ByteBuffer) each byte was its own native peek; one bulk + * copy first is far cheaper. + */ + private fun heapReader(reader: ByteReader): ByteReader = + if (reader is ArrayByteReader) reader else ArrayByteReader(ByteArray(reader.remaining()).also { reader.get(it) }) + private fun parseDraws(draws: ByteReader?): Array { if (((draws == null) || (draws.remaining() < 1))) { return NO_DRAWS } - val cursor: ByteReader = draws + val cursor: ByteReader = heapReader(draws) var out: MutableList = ArrayList() while ((cursor.remaining() >= 1)) { var kind: Int = (cursor.get() and 0xff) @@ -253,8 +261,11 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr private var shellScene: ShellScene? = null fun setShellScene(reader: ByteReader) { - val scene = ShellScene.decode(reader) - lock.withLock { shellScene = scene } + val scene = ShellScene.decode(heapReader(reader)) + lock.withLock { + shellScene = scene + changedSinceComposite = true + } } private class Surface { @@ -284,6 +295,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr /** Surface-local deferred draws of the retained content (already baked into pixels). */ @JvmField var draws: Array = NO_DRAWS + /** + * [draws]' retained selections in screen coordinates, as built for [placedFor] at + * ([placedX], [placedY]). Reused by every composite until either changes, so the shell + * scene built from them (and its fingerprint) can be reused too. + */ + @JvmField var placedSelections: List = emptyList() + @JvmField var placedFor: Array? = null + @JvmField var placedX: Int = 0 + @JvmField var placedY: Int = 0 + constructor(id: String) { this.id = id } @@ -291,6 +312,8 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr private val lock = protocolPlatform().createLock() private var animatedPreview: ShellScene.AnimatedPreview? = null + /** The last composite's scene; reused while its inputs are unchanged (see compositeLocked). */ + private var lastScene: ShellScene? = null private var previewKey: String? = null private fun previewScene(scene: ShellScene, gray: ByteArray, key: String): ByteArray { @@ -318,6 +341,13 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr private var nextCompositeSeq: Long = 1 + /** + * Set by changes that take effect at the next composite (geometry, dim, depth, blanking, + * visibility, a new shell scene, content retained without compositing) and cleared by a + * composite that is kept (not a preview); see [isSurfaceCurrent]. + */ + private var changedSinceComposite: Boolean = false + /** Set the output frame size. Must be called before any surface work. */ fun configureScreen(width: Int, height: Int): Unit { if (((width <= 0) || (height <= 0))) { @@ -376,12 +406,27 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr surface.height = height surface.zOrder = zOrder surface.transparency = transparency + changedSinceComposite = true } } fun removeSurface(id: String): Unit { lock.withLock { surfaces.remove(id) + changedSinceComposite = true + } + } + + /** + * Whether a full-surface update carrying [fingerprint] would change nothing: the surface + * already retains that content and every other change has reached a kept composite. A + * submitter can then drop an identical repaint without flattening or sending it. + */ + fun isSurfaceCurrent(id: String, fingerprint: String): Boolean { + lock.withLock { + if (changedSinceComposite) return false + val surface = surfaces.get(id) ?: return false + return surface.fingerprint.isNotEmpty() && surface.fingerprint == fingerprint } } @@ -398,6 +443,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr lock.withLock { underlayDimBelowZOrder = belowZOrder underlayDim = maxOf(0, minOf(256, factor256)) + changedSinceComposite = true } } @@ -419,6 +465,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr throw IllegalArgumentException(("unknown surface " + id)) } surface.visible = visible + changedSinceComposite = true } } @@ -432,6 +479,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr lock.withLock { val surface = surfaces.get(id) ?: throw IllegalArgumentException(("unknown surface " + id)) surface.depth = depth + changedSinceComposite = true } } @@ -450,6 +498,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr fun setBlanked(blanked: Boolean): Unit { lock.withLock { this.blanked = blanked + changedSinceComposite = true } } @@ -596,6 +645,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr } surface.fingerprint = (if ((contentFingerprint == null)) "" else contentFingerprint) surface.draws = parsed + if (!composeAfter) changedSinceComposite = true return if (composeAfter) compositeLocked() else null } } @@ -618,8 +668,10 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr return null } val seq = nextCompositeSeq + val changed = changedSinceComposite val result = compositeLocked(includePreview = true) nextCompositeSeq = seq + changedSinceComposite = changed return result } } @@ -640,6 +692,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr } private fun compositeLocked(includePreview: Boolean = includePreviewInFrames): Composite { + changedSinceComposite = false var gray: ByteArray = ByteArray((screenWidth * screenHeight)) if (blanked) { animatedPreview?.player?.stop() @@ -666,9 +719,9 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr } blendLocked(gray, surface) var dim: Int = dimForLocked(surface) + selections.addAll(placedSelectionsLocked(surface)) for (draw in surface.draws) { - val selected = draw.selection - if (selected != null) { selections.add(selected.translated(surface.x, surface.y)); continue } + if (draw.selection != null) continue if (((dim < 256) && (draw.kind == ScreenDraw.KIND_IMAGE))) { continue } @@ -710,8 +763,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr } } val screenDepth = screenDepthLocked(ordered) - val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene(emptyList(), screenDepth = screenDepth) - else ShellScene(shellScene?.layers ?: emptyList(), selections, screenDepth) + val overlaid = surfaces.values.any { it.visible && it.zOrder > 1 } + val sceneLayers = if (overlaid) emptyList() else shellScene?.layers ?: emptyList() + val sceneSelections: List = if (overlaid) emptyList() else selections + // Built per composite, a scene rebuilt its fingerprint and resource lists every time even + // though its inputs (the decoded shell layers, cached placed selections) rarely change. + val previous = lastScene + val scene = if (previous != null && previous.layers === sceneLayers && previous.screenDepth == screenDepth && + sameElements(previous.selections, sceneSelections)) previous + else ShellScene(sceneLayers, sceneSelections, screenDepth) + lastScene = scene val sceneKey = fingerprint.append("|shell:").append(scene.fingerprint).toString() val preview = if (includePreview && (shellScene != null || selections.isNotEmpty())) { previewScene(scene, gray, sceneKey) @@ -727,6 +788,22 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene } } + private fun placedSelectionsLocked(surface: Surface): List { + if (surface.placedFor !== surface.draws || surface.placedX != surface.x || surface.placedY != surface.y) { + surface.placedSelections = surface.draws.mapNotNull { it.selection?.translated(surface.x, surface.y) } + surface.placedFor = surface.draws + surface.placedX = surface.x + surface.placedY = surface.y + } + return surface.placedSelections + } + + private fun sameElements(a: List, b: List): Boolean { + if (a.size != b.size) return false + for (i in a.indices) if (a[i] !== b[i]) return false + return true + } + private fun blendLocked(gray: ByteArray, surface: Surface): Unit { var srcX: Int = maxOf(0, -surface.x) var srcY: Int = maxOf(0, -surface.y) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt index 74ee8e57..ee4e8598 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt @@ -268,13 +268,16 @@ class FrameTimingsCore( get() = lock.withLock { exportDirty } /** The full export text (does not clear the dirty flag). */ - fun buildExport(): String = lock.withLock { buildExportLocked() } + fun buildExport(): String = formatExport(lock.withLock { exportSnapshotLocked() }) /** The export text if anything changed since the last take, else null; clears the dirty flag. */ - fun takeExport(): String? = lock.withLock { - if (!exportDirty) return null - exportDirty = false - buildExportLocked() + fun takeExport(): String? { + val snapshot = lock.withLock { + if (!exportDirty) return null + exportDirty = false + exportSnapshotLocked() + } + return formatExport(snapshot) } // --------------------------------------------------------------------- @@ -360,30 +363,72 @@ class FrameTimingsCore( ) } - private fun buildExportLocked(): String { + /** + * Frame state as of one moment, formatted into the export text outside the lock. Formatting + * the full export takes tens of milliseconds, and holding the lock for it stalled every + * thread recording a span meanwhile, the main thread included. + */ + private class FrameSnapshot( + val id: Int, + val reason: String, + val startedAtMs: Long, + val startedWallClockMs: Long, + val durationMs: Long, + val open: Boolean, + val outcome: String?, + val subtreeEndAtMs: Long, + val subtreeHasOpenFrame: Boolean, + val lines: List, + val children: List, + ) + + private class ExportSnapshot( + val wallClockMs: Long, + val statsSummary: String, + val slowest: List, + val recent: List, + ) + + private fun snapshotLocked(frame: Frame, now: Long): FrameSnapshot = FrameSnapshot( + frame.id, frame.reason, frame.startedAtMs, frame.startedWallClockMs, frame.durationMs(), frame.isOpen(), + frame.outcome, frame.subtreeEndAtMs(now), frame.subtreeHasOpenFrame(), frame.lines.toList(), + frame.children.map { snapshotLocked(it, now) }, + ) + + private fun exportSnapshotLocked(): ExportSnapshot { val now = platform.elapsedRealtimeMs() + // Stable sort, descending by whole-tree duration (matches Collections.sort). + val treeDurations = HashMap() + for (frame in slowestRoots) treeDurations[frame] = frame.subtreeEndAtMs(now) - frame.startedAtMs + slowestRoots.sortWith(compareByDescending { treeDurations.getValue(it) }) + while (slowestRoots.size > SLOWEST_ROOTS_KEPT) { + slowestRoots.removeAt(slowestRoots.size - 1) + } + return ExportSnapshot( + wallClockMs(), + statsSummaryLocked(), + slowestRoots.map { snapshotLocked(it, now) }, + recentRoots.map { snapshotLocked(it, now) }, + ) + } + + private fun formatExport(snapshot: ExportSnapshot): String { val out = StringBuilder(64 * 1024) - out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n') - out.append(statsSummaryLocked()).append('\n') + out.append("FrameTimings export at ").append(formatWallClock(snapshot.wallClockMs)).append('\n') + out.append(snapshot.statsSummary).append('\n') out.append('\n') out.append("Frames are trees: an input event's own frame is the root, and the renders it\n") out.append("caused are indented under it with offsets measured from the root's start.\n") out.append('\n') - // Stable sort, descending by whole-tree duration (matches Collections.sort). - slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs }) - while (slowestRoots.size > SLOWEST_ROOTS_KEPT) { - slowestRoots.removeAt(slowestRoots.size - 1) - } - out.append("=== slowest frames that reached the glasses ===\n") - for (frame in slowestRoots) { - appendTreeLocked(out, frame, frame, now) + for (frame in snapshot.slowest) { + appendTree(out, frame, frame, 0) } out.append("=== recent frames (oldest first) ===\n") - for (frame in recentRoots) { - appendTreeLocked(out, frame, frame, now) + for (frame in snapshot.recent) { + appendTree(out, frame, frame, 0) } return out.toString() } @@ -416,28 +461,22 @@ class FrameTimingsCore( } /** Print a frame and its descendants, all timed against root's start. */ - private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) { - var depth = 0 - var walk: Frame? = frame - while (walk !== root) { - depth++ - walk = walk!!.parent - } + private fun appendTree(out: StringBuilder, frame: FrameSnapshot, root: FrameSnapshot, depth: Int) { val indent = " ".repeat(depth) val startOffsetMs = frame.startedAtMs - root.startedAtMs out.append(indent).append("frame#").append(frame.id) .append(" [").append(frame.reason).append(']') if (frame === root) { out.append(" started ").append(formatWallClock(frame.startedWallClockMs)) - val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs - out.append(" duration ").append(frame.durationMs()).append("ms") + val totalMs = frame.subtreeEndAtMs - frame.startedAtMs + out.append(" duration ").append(frame.durationMs).append("ms") if (frame.children.isNotEmpty()) { out.append(" (tree ").append(totalMs).append("ms") - .append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')') + .append(if (frame.subtreeHasOpenFrame) ", still open" else "").append(')') } } else { out.append(" started +").append(startOffsetMs).append("ms") - .append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms")) + .append(" duration ").append(if (frame.open) "open" else (frame.durationMs.toString() + "ms")) } out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n') for (line in frame.lines) { @@ -445,7 +484,7 @@ class FrameTimingsCore( .append("ms [").append(line.thread).append("] ").append(line.message).append('\n') } for (child in frame.children) { - appendTreeLocked(out, child, root, now) + appendTree(out, child, root, depth + 1) } } } diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SharedGraphicsTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SharedGraphicsTest.kt index d2ef7b4a..c477b867 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SharedGraphicsTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SharedGraphicsTest.kt @@ -97,6 +97,46 @@ class SharedGraphicsTest { assertContentEquals(byteArrayOf(-56, -56, 0, -56, -56, 0), c.composite().gray) } + @Test + fun surfaceIsCurrentOnlyAfterItsContentAndEveryChangeWasComposited() { + val c = SurfaceCompositor() + c.configureScreen(3, 2) + c.configureSurface("app", 0, 0, 3, 2, 0, 0) + assertFalse(c.isSurfaceCurrent("app", "")) + c.applyAndComposite("app", ArrayByteReader(ByteArray(6) { 200.toByte() }), 0, 0, 3, 2, "a") + assertTrue(c.isSurfaceCurrent("app", "a")) + assertFalse(c.isSurfaceCurrent("app", "b")) + assertFalse(c.isSurfaceCurrent("missing", "a")) + // A preview composite is not kept, so it does not apply pending changes. + c.setUnderlayDim(1, 128) + assertFalse(c.isSurfaceCurrent("app", "a")) + c.previewComposite() + assertFalse(c.isSurfaceCurrent("app", "a")) + c.composite() + assertTrue(c.isSurfaceCurrent("app", "a")) + for (change in listOf<() -> Unit>( + { c.configureSurface("app", 1, 0, 3, 2, 0, 0) }, + { c.setSurfaceVisible("app", false) }, + { c.setSurfaceDepth("app", 2) }, + { c.setBlanked(false) }, + { c.removeSurface("other") }, + )) { + change() + assertFalse(c.isSurfaceCurrent("app", "a")) + c.composite() + assertTrue(c.isSurfaceCurrent("app", "a")) + } + // Submitting without compositing retains the content but leaves it unapplied. + c.submitSurface("app", ArrayByteReader(ByteArray(6)), 0, 0, 3, 2, "b") + assertFalse(c.isSurfaceCurrent("app", "b")) + c.composite() + assertTrue(c.isSurfaceCurrent("app", "b")) + // A resize discards the retained content. + c.configureSurface("app", 0, 0, 2, 2, 0, 0) + c.composite() + assertFalse(c.isSurfaceCurrent("app", "b")) + } + @Test fun pngChunksAndPixels() { val png = SharedScreenshots.encode4BitGrayPng(byteArrayOf(0, 16, -1, 32, 48, 64), 3, 2)