diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt index 83c294be..dd03f339 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt @@ -35,6 +35,8 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { companion object { private const val TAG = "FaceclawNotify" private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName" + /** MessagingStyle's conversation icon (Notification.EXTRA_CONVERSATION_ICON, hidden). */ + private const val EXTRA_CONVERSATION_ICON = "android.conversationIcon" /** * How long notification access can be granted with no listener * connected before the watchdog forces a rebind: longer than a @@ -47,6 +49,8 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { private const val LISTENER_STUCK_MS = 30_000L /** Notifications posted while disconnected alert on reconnect only if this recent. */ private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L + /** Large icons are drawn at no more than this many pixels a side before scaling down. */ + private const val LARGE_ICON_MAX_SOURCE = 512 /** The connected listener; null until onListenerConnected and after onListenerDisconnected. */ @Volatile @@ -366,6 +370,48 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground) } + /** + * One active notification's large icon (largeIconOf: a sender's avatar, a site's logo) + * scaled to fit size x size as a gray packet, a light background shown as its negative + * like a picture's; empty when the notification or its icon is unavailable. + */ + @JvmStatic + fun getNotificationLargeIconGray(key: String?, size: Int, gamma: Float, dither: Boolean): ByteArray { + val service = activeService + if (service == null || size <= 0) { + return ByteArray(0) + } + val statusBarNotification = findActiveNotificationByKey(service, key) + val icon = statusBarNotification?.notification?.let { largeIconOf(it) } ?: return ByteArray(0) + try { + val drawable = icon.loadDrawable(service) ?: return ByteArray(0) + val width = if (drawable.intrinsicWidth > 0) Math.min(drawable.intrinsicWidth, LARGE_ICON_MAX_SOURCE) else size * 4 + val height = if (drawable.intrinsicHeight > 0) Math.min(drawable.intrinsicHeight, LARGE_ICON_MAX_SOURCE) else size * 4 + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + drawable.setBounds(0, 0, width, height) + drawable.draw(Canvas(bitmap)) + return ImageFileLoader.bitmapToGrayPacket(halveToward(bitmap, size, size), size, size, gamma, dither, true) + } catch (t: Throwable) { + Log.w(TAG, "failed to load large icon for " + statusBarNotification.packageName, t) + return ByteArray(0) + } + } + + /** + * One active notification's custom layout (hasCustomView in its JSON) drawn to fit + * maxWidth x maxHeight as a gray packet; see NotificationCustomViews.renderGray. Empty + * when the notification or its layout is unavailable. + */ + @JvmStatic + fun getNotificationCustomViewGray(key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray { + val service = activeService ?: return ByteArray(0) + val statusBarNotification = findActiveNotificationByKey(service, key) + val notification = statusBarNotification?.notification ?: return ByteArray(0) + val hasPicture = notification.extras?.let { hasNotificationPicture(it) } ?: false + return NotificationCustomViews.renderGray( + service, statusBarNotification, !hasPicture, maxWidth, maxHeight, gamma, dither) + } + @JvmStatic fun getActiveNotificationsJson(maxNotifications: Int): String { val service = activeService @@ -672,6 +718,32 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { return null } + /** + * The picture a notification shows beside its text: its large icon (Gmail's sender + * avatar or logo), else from Android 11 its conversation's icon, else the icon of its + * latest message's sender. + */ + @Suppress("DEPRECATION") + private fun largeIconOf(notification: Notification): Icon? { + notification.getLargeIcon()?.let { return it } + val extras = notification.extras + if (extras == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + return null + } + try { + val conversationIcon: Icon? = extras.getParcelable(EXTRA_CONVERSATION_ICON) + if (conversationIcon != null) { + return conversationIcon + } + val messages = extras.getParcelableArray(Notification.EXTRA_MESSAGES) ?: return null + return Notification.MessagingStyle.Message.getMessagesFromBundleArray(messages) + .lastOrNull()?.senderPerson?.icon + } catch (t: Throwable) { + Log.w(TAG, "failed to read conversation icons", t) + return null + } + } + /** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */ private fun softwareBitmap(bitmap: Bitmap): Bitmap { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) { @@ -687,7 +759,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { * (NotificationIconLoader.renderArgb), one filtered pass from a much larger source * samples too sparsely: a graph's thin lines break up into dots. */ - private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap { + internal fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap { var bitmap = source while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) { val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true) @@ -759,6 +831,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { out.put("lines", JSONArray()) out.put("hasPicture", false) } + out.put("hasLargeIcon", largeIconOf(notification) != null) + val hasCustomView = NotificationCustomViews.hasCustomView(notification) + out.put("hasCustomView", hasCustomView) + val customText = JSONArray() + // Only needed (and only worth inflating the layout for) when there's no title or text to show instead. + if (hasCustomView && out.optString("title").isEmpty() && out.optString("text").isEmpty()) { + for (text in NotificationCustomViews.texts(service, statusBarNotification)) { + customText.put(text) + } + } + out.put("customText", customText) val actionsJson = JSONArray() val actions: Array? = notification.actions diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationCustomViews.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationCustomViews.kt new file mode 100644 index 00000000..42773246 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationCustomViews.kt @@ -0,0 +1,200 @@ +package com.faceclaw.app + +import android.app.Notification +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Picture +import android.os.Build +import android.service.notification.StatusBarNotification +import android.util.Log +import android.view.View +import android.view.ViewGroup +import android.widget.FrameLayout +import android.widget.RemoteViews +import android.widget.TextView + +import kotlin.math.ceil +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +/** + * A notification's custom layout (RemoteViews), which can carry what its standard fields leave + * out: Dexcom's G7 notification has no title or text at all, just a layout with the glucose + * reading and trend arrow. Inflated here the way the shade does it, for its text (a stand-in + * title in lists) and to draw it as a picture. + */ +internal object NotificationCustomViews { + private const val TAG = "FaceclawNotifyView" + /** Set by Notification.Builder when the app gave it a custom layout (Notification.EXTRA_CONTAINS_CUSTOM_VIEW, hidden). */ + private const val EXTRA_CONTAINS_CUSTOM_VIEW = "android.contains.customView" + private const val VIEW_CACHE_MAX_ENTRIES = 8 + private const val TEXT_CACHE_MAX_ENTRIES = 64 + /** Layouts are cut off below this height, in dp; the shade caps an expanded notification at 256. */ + private const val MAX_HEIGHT_DP = 320 + + /** Inflated layouts by notification post and layout; null when inflating failed, so it isn't retried every paint. */ + private val views = Lru(VIEW_CACHE_MAX_ENTRIES) + private val texts = Lru>(TEXT_CACHE_MAX_ENTRIES) + + /** + * Whether the app gave the notification a custom layout. The flag tells one apart from the + * standard template that Notification.Builder stores in contentView for apps targeting + * Android 6 and older, which only repeats the title and text. + */ + @Suppress("DEPRECATION") + fun hasCustomView(notification: Notification): Boolean { + val extras = notification.extras ?: return false + return extras.getBoolean(EXTRA_CONTAINS_CUSTOM_VIEW) + && (notification.contentView != null || notification.bigContentView != null || notification.headsUpContentView != null) + } + + /** + * The text of the custom layout as collapsed in the shade, one entry per visible TextView in + * layout order; empty without a custom layout. Cached per post of the notification. + */ + @Synchronized + fun texts(context: Context, statusBarNotification: StatusBarNotification): List { + val notification = statusBarNotification.notification ?: return emptyList() + val (name, layout) = layoutFor(notification, expanded = false) ?: return emptyList() + return texts.getOrPut(postKey(statusBarNotification, name)) { + inflate(context, statusBarNotification, name, layout)?.let { collectTexts(it, ArrayList()) } ?: emptyList() + } + } + + /** + * The custom layout drawn as a gray packet fitting maxWidth x maxHeight; empty without one + * or when it can't be drawn. It is laid out maxWidth dp wide and shown at no more than a + * glasses pixel per dp, where the shade's text sizes land near the glasses' UI font, then + * trimmed to what it draws. The collapsed layout is used when the notification has a + * picture to show under it (an expanded layout tends to repeat the picture, as Dexcom's + * does), else the expanded one. Ink drawn for a light background comes out light on dark. + */ + @Synchronized + fun renderGray( + context: Context, statusBarNotification: StatusBarNotification, expanded: Boolean, + maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, + ): ByteArray { + val notification = statusBarNotification.notification ?: return ByteArray(0) + if (maxWidth <= 0 || maxHeight <= 0) return ByteArray(0) + val (name, layout) = layoutFor(notification, expanded) ?: return ByteArray(0) + val view = inflate(context, statusBarNotification, name, layout) ?: return ByteArray(0) + try { + val density = context.resources.displayMetrics.density + val widthPx = max(1, (maxWidth * density).roundToInt()) + val maxHeightPx = max(1, (MAX_HEIGHT_DP * density).roundToInt()) + view.measure( + View.MeasureSpec.makeMeasureSpec(widthPx, View.MeasureSpec.EXACTLY), + View.MeasureSpec.makeMeasureSpec(0, View.MeasureSpec.UNSPECIFIED)) + if (view.measuredHeight > maxHeightPx) { + view.measure( + View.MeasureSpec.makeMeasureSpec(widthPx, View.MeasureSpec.EXACTLY), + View.MeasureSpec.makeMeasureSpec(maxHeightPx, View.MeasureSpec.AT_MOST)) + } + val heightPx = min(view.measuredHeight, maxHeightPx) + if (heightPx <= 0) return ByteArray(0) + view.layout(0, 0, widthPx, heightPx) + + val drawn = drawView(view, widthPx, heightPx) + val pixels = IntArray(widthPx * heightPx) + drawn.getPixels(pixels, 0, widthPx, 0, 0, widthPx, heightPx) + drawn.recycle() + val bounds = GrayPacket.inkBounds(pixels, widthPx, heightPx) ?: return ByteArray(0) + val cropWidth = bounds[2] - bounds[0] + val cropHeight = bounds[3] - bounds[1] + val cropped = IntArray(cropWidth * cropHeight) + for (y in 0 until cropHeight) { + System.arraycopy(pixels, (bounds[1] + y) * widthPx + bounds[0], cropped, y * cropWidth, cropWidth) + } + GrayPacket.lightenDarkInk(cropped, cropWidth, cropHeight) + val art = Bitmap.createBitmap(cropped, cropWidth, cropHeight, Bitmap.Config.ARGB_8888) + // A dp per glasses pixel at most: bitmapToGrayPacket otherwise blows a narrow layout up to the box. + val fitWidth = min(maxWidth, ceil(cropWidth / density).toInt()) + val fitHeight = min(maxHeight, ceil(cropHeight / density).toInt()) + Log.i(TAG, "rendered $name layout of ${statusBarNotification.packageName}: ${cropWidth}x$cropHeight px" + + " into ${fitWidth}x$fitHeight") + return ImageFileLoader.bitmapToGrayPacket( + FaceclawMediaNotificationListenerService.halveToward(art, fitWidth, fitHeight), + fitWidth, fitHeight, gamma, dither, true) + } catch (t: Throwable) { + Log.w(TAG, "failed to draw the $name layout of ${statusBarNotification.packageName}", t) + return ByteArray(0) + } + } + + /** + * The custom layout to use, with the field it came from: the collapsed one (contentView) + * or the expanded one (bigContentView), falling back on the other, then on the heads-up one. + */ + @Suppress("DEPRECATION") + private fun layoutFor(notification: Notification, expanded: Boolean): Pair? { + if (!hasCustomView(notification)) return null + val collapsed = notification.contentView?.let { "content" to it } + val big = notification.bigContentView?.let { "big" to it } + val first = if (expanded) big ?: collapsed else collapsed ?: big + return first ?: notification.headsUpContentView?.let { "headsUp" to it } + } + + /** A notification is reposted (with a new postTime) whenever its app updates it. */ + private fun postKey(statusBarNotification: StatusBarNotification, layoutName: String): String = + "${statusBarNotification.key}|${statusBarNotification.postTime}|$layoutName" + + private fun inflate(context: Context, statusBarNotification: StatusBarNotification, name: String, layout: RemoteViews): View? = + views.getOrPut(postKey(statusBarNotification, name)) { + try { + // RemoteViews loads the layout and its resources from the posting app's package. + layout.apply(context, FrameLayout(context)) + } catch (t: Throwable) { + Log.w(TAG, "failed to inflate the $name layout of ${statusBarNotification.packageName}", t) + null + } + } + + private fun collectTexts(view: View, out: MutableList): List { + if (view.visibility != View.VISIBLE) return out + if (view is TextView) { + val text = view.text?.toString()?.replace(Regex("\\s+"), " ")?.trim() ?: "" + if (text.isNotEmpty()) out.add(text) + } else if (view is ViewGroup) { + for (index in 0 until view.childCount) { + collectTexts(view.getChildAt(index), out) + } + } + return out + } + + /** + * Draws a laid-out view to a software bitmap. From Android 9 this goes through a Picture, + * which (unlike a bitmap's Canvas) accepts the hardware bitmaps a layout's images can + * arrive as; Bitmap.createBitmap then renders it on the GPU and reads it back if it has to. + */ + private fun drawView(view: View, width: Int, height: Int): Bitmap { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + val picture = Picture() + view.draw(picture.beginRecording(width, height)) + picture.endRecording() + return Bitmap.createBitmap(picture, width, height, Bitmap.Config.ARGB_8888) + } + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + view.draw(Canvas(bitmap)) + return bitmap + } + + /** A small LRU map; values may be null. Callers hold the object's lock. */ + private class Lru(private val maxEntries: Int) { + private val entries = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean = size > maxEntries + } + + fun getOrPut(key: String, compute: () -> V): V { + if (entries.containsKey(key)) { + @Suppress("UNCHECKED_CAST") + return entries[key] as V + } + val value = compute() + entries[key] = value + return value + } + } +} diff --git a/CHANGELOG b/CHANGELOG index b3931c8a..46753062 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,9 +2,10 @@ ## 0.8.6 - Future -+ Add a distance unit option -+ Make BLE benchmark work on iOS -+ Ensure connection is re-set to 2M on reconnect +* Add a distance unit option +* Make BLE benchmark work on iOS +* Ensure connection is re-set to 2M on reconnect +* Add support for embedded icons and custom layouts in Android notifications ## 0.8.5 - 2026-10-25 diff --git a/app/assistant/system-tools.ts b/app/assistant/system-tools.ts index b6745d83..4e3f17f3 100644 --- a/app/assistant/system-tools.ts +++ b/app/assistant/system-tools.ts @@ -244,8 +244,8 @@ function formatEvent(event: CalendarEvent): string { return `- ${event.title || "(untitled)"} — ${when}${location}`; } -function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string { - const title = notification.title || "(no title)"; +function formatNotification(notification: { key: string; appName: string; title: string; text: string; customText?: string[] }): string { + const title = notification.title || notification.customText?.join(" ") || "(no title)"; const body = notification.text ? `: ${notification.text}` : ""; return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`; } diff --git a/app/native/notification-icons.ios.ts b/app/native/notification-icons.ios.ts index c908c838..2aa15f7c 100644 --- a/app/native/notification-icons.ios.ts +++ b/app/native/notification-icons.ios.ts @@ -4,7 +4,7 @@ import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-cli import { rememberNotificationSources } from './notification-sources' import type { AndroidNotification } from './notification-types' export type { AndroidNotification, AndroidNotificationAction } from './notification-types' -export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons' +export type { NotificationIconsResult, NotificationIconResult, NotificationImageResult } from './notification-icons' export const ALL_NOTIFICATIONS = 0x7fffffff let active: AncsClient | null = null @@ -61,7 +61,13 @@ export function readNotificationIconByKey(key: string, _allowStale: boolean) { const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key) return {icon: notification ? categoryIcon(notification.category) : null,stale:false} } -// ANCS carries no attachments. +// ANCS carries no attachments, avatars or custom layouts. export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) { - return {picture: null,stale:false} + return {image: null,stale:false} +} +export function readNotificationLargeIcon(_notification: AndroidNotification, _size: number, _allowStale: boolean) { + return {image: null,stale:false} +} +export function readNotificationCustomView(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) { + return {image: null,stale:false} } diff --git a/app/native/notification-icons.ts b/app/native/notification-icons.ts index aa652096..065e6747 100644 --- a/app/native/notification-icons.ts +++ b/app/native/notification-icons.ts @@ -23,11 +23,6 @@ let cachedKeys: string[] = []; let cachedAtMs = 0; const keyedIconCache = new Map(); const KEYED_ICON_CACHE_MAX = 128; -/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */ -const pictureCache = new Map(); -const PICTURE_CACHE_MAX = 8; let notificationListenerProxy: any = null; const notificationPostedListeners = new Set<(notificationKey: string) => void>(); const notificationRemovedListeners = new Set<(notificationKey: string) => void>(); @@ -159,48 +154,86 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not return { icon: icon?.clone() ?? null, stale: false }; } -export type NotificationPictureResult = { +export type NotificationImageResult = { /** Shared with the cache: blit it, don't draw on it. */ - picture: GrayImage | null; + image: GrayImage | null; /** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */ stale: boolean; }; +/** + * Images belonging to one notification (its picture, large icon or custom + * layout), fetched to fit a box, under the allow-stale contract of + * readNotificationIconByKey. Cached per post of the notification: an ongoing + * notification updated in place (Dexcom redraws its graph and reading every + * few minutes) fires no posted event, but its postTime moves. ICON_CACHE_MS + * is the backstop here too. Under allowStale an earlier fetch stands in, if + * it fits. + */ +class NotificationImageCache { + private readonly entries = new Map(); + + constructor(private readonly maxEntries: number, private readonly span: string) {} + + read( + notification: AndroidNotification, + maxWidth: number, + maxHeight: number, + allowStale: boolean, + fetch: (width: number, height: number) => GrayImage | null, + ): NotificationImageResult { + const width = Math.round(maxWidth); + const height = Math.round(maxHeight); + if (!global.isAndroid || width <= 0 || height <= 0) return { image: null, stale: false }; + + const now = Date.now(); + const cached = this.entries.get(notification.key); + if ( + cached && cached.postTime === notification.postTime && cached.maxWidth === width + && cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS + ) { + return { image: cached.image, stale: false }; + } + if (allowStale) { + const fits = cached?.image && cached.image.width <= width && cached.image.height <= height; + return { image: fits ? cached!.image : null, stale: true }; + } + + const image = spanCurrent(this.span, () => fetch(width, height)); + this.entries.delete(notification.key); + this.entries.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, image, atMs: now }); + while (this.entries.size > this.maxEntries) { + const oldestKey = this.entries.keys().next().value; + if (oldestKey === undefined) break; + this.entries.delete(oldestKey); + } + return { image, stale: false }; + } +} + +/** Pictures and custom layouts are the size of a detail view, not icons: keep just the recently viewed few. */ +const pictureCache = new NotificationImageCache(8, "fetch-notification-picture"); +const customViewCache = new NotificationImageCache(8, "fetch-notification-custom-view"); +/** Large icons show on every list card drawn, so keep a screenful and then some. */ +const largeIconCache = new NotificationImageCache(32, "fetch-notification-large-icon"); +/** Avatars and logos: photographic tone, but undithered, which at icon size reads as noise. */ +const LARGE_ICON_TONE = { gamma: PHOTO_TONE.gamma, dither: false }; + /** * The picture attached to a notification (hasPicture), scaled to fit * maxWidth x maxHeight with the photo tone and inverted if it is drawn on a - * light background (GrayPacket.hasLightBackground), under the allow-stale contract of - * readNotificationIconByKey. Cached per post of the notification: an ongoing - * notification updated in place (Dexcom redraws its graph every few minutes) - * fires no posted event, but its postTime moves. ICON_CACHE_MS is the - * backstop here too. Under allowStale an earlier fetch stands in, if it fits. + * light background (GrayPacket.hasLightBackground); see NotificationImageCache. */ export function readNotificationPicture( notification: AndroidNotification, maxWidth: number, maxHeight: number, allowStale: boolean, -): NotificationPictureResult { - const width = Math.round(maxWidth); - const height = Math.round(maxHeight); - if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) { - return { picture: null, stale: false }; - } - - const now = Date.now(); - const cached = pictureCache.get(notification.key); - if ( - cached && cached.postTime === notification.postTime && cached.maxWidth === width - && cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS - ) { - return { picture: cached.picture, stale: false }; - } - if (allowStale) { - const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height; - return { picture: fits ? cached!.picture : null, stale: true }; - } - - const picture = spanCurrent("fetch-notification-picture", () => +): NotificationImageResult { + if (!notification.hasPicture) return { image: null, stale: false }; + return pictureCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) => grayImageFromPacket( com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray( notification.key, @@ -212,16 +245,56 @@ export function readNotificationPicture( // lit panel on the glasses: show it light-on-dark. true, ), - ), - ); - pictureCache.delete(notification.key); - pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now }); - while (pictureCache.size > PICTURE_CACHE_MAX) { - const oldestKey = pictureCache.keys().next().value; - if (oldestKey === undefined) break; - pictureCache.delete(oldestKey); - } - return { picture, stale: false }; + )); +} + +/** + * A notification's large icon (hasLargeIcon: a sender's avatar, a site's + * logo), scaled to fit size x size, a light background inverted as for + * pictures; see NotificationImageCache. + */ +export function readNotificationLargeIcon( + notification: AndroidNotification, + size: number, + allowStale: boolean, +): NotificationImageResult { + if (!notification.hasLargeIcon) return { image: null, stale: false }; + return largeIconCache.read(notification, size, size, allowStale, (width) => + grayImageFromPacket( + com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationLargeIconGray( + notification.key, + width, + LARGE_ICON_TONE.gamma, + LARGE_ICON_TONE.dither, + ), + )); +} + +/** + * A notification's own layout (hasCustomView) drawn as the phone's shade + * draws it, at most a pixel per dp and trimmed to what it draws, so usually + * smaller than maxWidth x maxHeight; dark-on-light layouts come out + * light-on-dark. See NotificationCustomViews.renderGray and + * NotificationImageCache. + */ +export function readNotificationCustomView( + notification: AndroidNotification, + maxWidth: number, + maxHeight: number, + allowStale: boolean, +): NotificationImageResult { + if (!notification.hasCustomView) return { image: null, stale: false }; + return customViewCache.read(notification, maxWidth, maxHeight, allowStale, (width, height) => + grayImageFromPacket( + // Flat UI tone: it is mostly text and icons. + com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationCustomViewGray( + notification.key, + width, + height, + 1, + false, + ), + )); } export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] { @@ -354,6 +427,9 @@ function normalizeNotification(value: any): AndroidNotification | null { actions, foregroundService: Boolean(value.foregroundService), hasPicture: Boolean(value.hasPicture), + hasLargeIcon: Boolean(value.hasLargeIcon), + hasCustomView: Boolean(value.hasCustomView), + customText: Array.isArray(value.customText) ? value.customText.map((text: unknown) => String(text)).filter(Boolean) : [], }; } diff --git a/app/native/notification-types.ts b/app/native/notification-types.ts index 78d35ac0..0501a1a7 100644 --- a/app/native/notification-types.ts +++ b/app/native/notification-types.ts @@ -9,6 +9,15 @@ export type AndroidNotification = { foregroundService?: boolean; /** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */ hasPicture?: boolean; + /** Carries a large icon (Android: a sender's avatar or logo), fetched separately by readNotificationLargeIcon. */ + hasLargeIcon?: boolean; + /** Has its own layout (Android RemoteViews), drawn by readNotificationCustomView. */ + hasCustomView?: boolean; + /** + * The custom layout's text, one entry per text view, when the notification has no title or + * text of its own (Dexcom's glucose reading and units). + */ + customText?: string[]; /** Full message size reported by iOS, even when the fetched text is bounded. */ messageSize?: number; }; diff --git a/app/ui/notifications.ts b/app/ui/notifications.ts index 8f05a1b6..905d5d1c 100644 --- a/app/ui/notifications.ts +++ b/app/ui/notifications.ts @@ -9,10 +9,13 @@ import { dismissNotification, invokeNotificationAction, readActiveNotifications, + readNotificationCustomView, readNotificationIconByKey, + readNotificationLargeIcon, readNotificationPicture, type AndroidNotification, type AndroidNotificationAction, + type NotificationImageResult, } from "../native/notification-icons"; import { setNotificationSourceEnabled } from "../native/notification-sources"; import { notificationEmptyMessage } from "../native/notification-access"; @@ -45,13 +48,18 @@ const DETAIL_TITLE_Y = 21; /** Gaps below the title's line box and below the icon/app-name header row. */ const DETAIL_HEADER_GAP = 3; const DETAIL_BODY_GAP = 4; -/** Text lines an attached picture leaves room for above it (when there are that many); it takes the rest. */ +/** Untitled text lines that attached images (custom layout, picture) leave room for above them, when there are that many; they take the rest. */ const DETAIL_PICTURE_TEXT_LINES = 2; /** Gap between the text and an attached picture, and the space kept below the picture. */ const DETAIL_PICTURE_GAP = 6; const DETAIL_PICTURE_BOTTOM_MARGIN = 12; /** An attached picture with less height than this to fit in (a large UI font) is left out. */ const DETAIL_PICTURE_MIN_HEIGHT = 32; +/** Likewise a custom layout; Dexcom's reading is about 40 high. */ +const DETAIL_CUSTOM_VIEW_MIN_HEIGHT = 16; +/** A large icon (sender's avatar, site's logo): at the right of a list card, left of the detail view's title. */ +const LARGE_ICON_SIZE = 32; +const LARGE_ICON_GAP = 8; // Disable-source confirmation: actions on top, the explanation below them. const CONFIRM_X = 24; const CONFIRM_Y = 8; @@ -67,14 +75,27 @@ function iconForNotification(key: string): GrayImage | null { return icon; } -/** Attached picture for a paint pass, fitting maxWidth x maxHeight; allow-stale like iconForNotification. */ -function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null { - if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null; - const { picture, stale } = readNotificationPicture(notification, maxWidth, maxHeight, renderPassAllowsStaleData()); - if (stale) { +/** A notification image read for a paint pass (allow-stale like iconForNotification). */ +function imageForPaint( + read: (allowStale: boolean) => NotificationImageResult, +): NotificationImageResult { + const result = read(renderPassAllowsStaleData()); + if (result.stale) { noteStaleDataUsed(); } - return picture; + return result; +} + +/** Attached picture for a paint pass, fitting maxWidth x maxHeight. */ +function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null { + if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null; + return imageForPaint((allowStale) => readNotificationPicture(notification, maxWidth, maxHeight, allowStale)).image; +} + +/** Large icon for a paint pass, LARGE_ICON_SIZE square at most. */ +function largeIconForNotification(notification: AndroidNotification): GrayImage | null { + if (!notification.hasLargeIcon) return null; + return imageForPaint((allowStale) => readNotificationLargeIcon(notification, LARGE_ICON_SIZE, allowStale)).image; } type CardLayout = { @@ -152,7 +173,8 @@ export class NotificationsListLayer implements Layer { // Icons are only resolved for cards actually drawn, so a long list // does not fetch icons for everything below the fold. const icon = iconForNotification(layout.notification.key); - drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon); + const largeIcon = largeIconForNotification(layout.notification); + drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon, largeIcon); } cursorY += layout.height + CARD_GAP; if (cursorY > listBottom + 80) break; @@ -356,8 +378,10 @@ function buildNotificationCardLayout( font: UiFont, notification: AndroidNotification, selected: boolean, - cardTextWidth: number, + fullTextWidth: number, ): CardLayout { + // The large icon sits at the card's right, beside all its text. + const cardTextWidth = notification.hasLargeIcon ? fullTextWidth - LARGE_ICON_SIZE - LARGE_ICON_GAP : fullTextWidth; const lines: string[] = []; const time = formatRelativeTime(notification.postTime); const title = notificationTitle(notification); @@ -383,7 +407,7 @@ function buildNotificationCardLayout( return { notification, lines, - height: Math.max(ICON_SIZE + 18, 12 + lines.length * lineStep(font)), + height: Math.max(ICON_SIZE + 18, notification.hasLargeIcon ? LARGE_ICON_SIZE + 16 : 0, 12 + lines.length * lineStep(font)), }; } @@ -397,6 +421,7 @@ function drawNotificationCard( selected: boolean, focused: boolean, icon: GrayImage | null, + largeIcon: GrayImage | null, ): void { // Match the shared menu highlight: faint border + black fill when // unselected; bright border when selected; the non-black fill only appears @@ -408,6 +433,9 @@ function drawNotificationCard( if (icon) { image.bitBlt(icon, x + 10, y + 8, { transparentZero: true }); } + if (largeIcon) { + image.bitBlt(largeIcon, x + width - 10 - LARGE_ICON_SIZE + ((LARGE_ICON_SIZE - largeIcon.width) >> 1), y + 8); + } for (let index = 0; index < layout.lines.length; index++) { const value = index === 0 ? 140 : selected ? 235 : 185; image.drawText(font, x + CARD_TEXT_X, y + 7 + index * lineStep(font), layout.lines[index]!, value); @@ -437,18 +465,35 @@ function drawDetailContent( ): void { const contentX = NOTIFICATION_CONTENT_X; const contentWidth = notificationContentWidth(width); - const bodyTop = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`); + const step = lineStep(font); + const bottom = height - DETAIL_PICTURE_BOTTOM_MARGIN; + let top = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`); + + // The title, beside the large icon (a sender's avatar, a site's logo) when + // there is one. A picture or custom layout can be the whole notification + // (Dexcom's has no text), so either stands in for the "(untitled)" placeholder. + const hasImages = Boolean(notification.hasPicture || notification.hasCustomView); + const title = notification.title || (hasImages ? "" : "(untitled)"); + const avatarSize = notification.hasLargeIcon ? LARGE_ICON_SIZE : 0; + const titleX = avatarSize ? contentX + avatarSize + LARGE_ICON_GAP : contentX; + const titleWidth = contentX + contentWidth - titleX; + const titleLines = title ? wrapText(font, title, titleWidth) : []; + let titleHeight = 0; + if (avatarSize) { + const avatar = largeIconForNotification(notification); + if (avatar) image.bitBlt(avatar, contentX + ((avatarSize - avatar.width) >> 1), top + ((avatarSize - avatar.height) >> 1)); + titleHeight = avatarSize; + } + if (titleLines.length) { + // A title shorter than the avatar is centered on it. + const titleTop = top + Math.max(0, (avatarSize - titleLines.length * step) >> 1); + const shown = drawClippedLines(image, font, titleLines, titleX, titleTop, titleWidth, ((bottom - top) / step) | 0, 230); + titleHeight = Math.max(titleHeight, shown * step); + } const lines: string[] = []; - // A picture can be the whole notification (Dexcom's has no text), so it - // stands in for the "(untitled)" placeholder. - const hasTitle = Boolean(notification.title) || !notification.hasPicture; - if (hasTitle) { - lines.push(...wrapText(font, notification.title || "(untitled)", contentWidth)); - } const body = detailNotificationBody(notification); if (body) { - if (lines.length) lines.push(""); lines.push(...wrapText(font, body, contentWidth)); } const meta = [notification.subText, notification.infoText, notification.summaryText].filter(Boolean).join(" "); @@ -456,36 +501,65 @@ function drawDetailContent( if (lines.length) lines.push(""); lines.push(...wrapText(font, meta, contentWidth)); } + // Text follows the title after a blank line (just a gap beside an avatar); images after a gap. + if (titleHeight) { + top += titleHeight + (lines.length && !avatarSize ? step : DETAIL_PICTURE_GAP); + } - const step = lineStep(font); - const textValue = (index: number) => index === 0 && hasTitle ? 230 : 190; - const reservedTextHeight = Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step; - const picture = pictureForNotification(notification, contentWidth, - height - DETAIL_PICTURE_BOTTOM_MARGIN - bodyTop - reservedTextHeight - (reservedTextHeight ? DETAIL_PICTURE_GAP : 0)); - if (picture) { - // The text gets whatever the picture leaves, above it, with its last - // shown line marked when some is cut off. - const roomForText = height - DETAIL_PICTURE_BOTTOM_MARGIN - picture.height - DETAIL_PICTURE_GAP - bodyTop; - const shownLines = Math.min(lines.length, Math.max(0, (roomForText / step) | 0)); - for (let index = 0; index < shownLines; index++) { - const line = index === shownLines - 1 && lines.length > shownLines - ? truncateText(font, lines[index] + "...", contentWidth) : lines[index]!; - image.drawText(font, contentX, bodyTop + index * step, line, textValue(index)); + // The custom layout, then the picture, take what is left below the title, + // or below the first few lines of text when there is no title. + const reservedTextHeight = titleHeight ? 0 : Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step; + const roomForImages = bottom - top - (reservedTextHeight ? reservedTextHeight + DETAIL_PICTURE_GAP : 0); + let customView: GrayImage | null = null; + if (notification.hasCustomView && roomForImages >= DETAIL_CUSTOM_VIEW_MIN_HEIGHT) { + const result = imageForPaint((allowStale) => readNotificationCustomView(notification, contentWidth, roomForImages, allowStale)); + customView = result.image; + if (!customView && !result.stale && notification.customText?.length) { + // It couldn't be drawn: its text is better than nothing. + lines.unshift(...wrapText(font, notification.customText.join(" "), contentWidth), ...(lines.length ? [""] : [])); } - image.bitBlt(picture, contentX, bodyTop + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0)); - return; } + const picture = pictureForNotification(notification, contentWidth, + roomForImages - (customView ? customView.height + DETAIL_PICTURE_GAP : 0)); - const maxLines = Math.max(1, ((height - bodyTop - step) / step) | 0); - for (let index = 0; index < Math.min(lines.length, maxLines); index++) { - const line = lines[index]!; - image.drawText(font, contentX, bodyTop + index * step, line, textValue(index)); + // The text gets whatever the images leave, above them, with its last shown + // line marked when some is cut off. + const imagesHeight = (customView ? customView.height : 0) + (picture ? picture.height : 0) + + (customView && picture ? DETAIL_PICTURE_GAP : 0); + const roomForText = bottom - top - (imagesHeight ? imagesHeight + DETAIL_PICTURE_GAP : 0); + const shownLines = drawClippedLines(image, font, lines, contentX, top, contentWidth, (roomForText / step) | 0, 190); + let imageTop = top + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0); + if (customView) { + image.bitBlt(customView, contentX, imageTop); + imageTop += customView.height + DETAIL_PICTURE_GAP; } - if (lines.length > maxLines) { - image.drawText(font, contentX, height - 24, "...", 140); + if (picture) { + image.bitBlt(picture, contentX, imageTop); } } +/** + * Draw up to maxLines of lines from (x, y), marking the last one drawn with + * "..." when some are left out. Returns how many were drawn. + */ +function drawClippedLines( + image: GrayImage, + font: UiFont, + lines: string[], + x: number, + y: number, + width: number, + maxLines: number, + value: number, +): number { + const shown = Math.min(lines.length, Math.max(0, maxLines)); + for (let index = 0; index < shown; index++) { + const line = index === shown - 1 && lines.length > shown ? truncateText(font, lines[index] + "...", width) : lines[index]!; + image.drawText(font, x, y + index * lineStep(font), line, value); + } + return shown; +} + /** Left edge of a notification detail view's content column. */ export const NOTIFICATION_CONTENT_X = DETAIL_CONTENT_X; @@ -577,5 +651,6 @@ function detailNotificationBody(notification: AndroidNotification): string { } function notificationTitle(notification: AndroidNotification): string { - return notification.title || notification.text || notification.summaryText || notification.appName || notification.packageName || "(untitled)"; + return notification.title || notification.text || notification.customText?.join(" ") || notification.summaryText + || notification.appName || notification.packageName || "(untitled)"; } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt index 172663f8..9e0f2b72 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt @@ -91,6 +91,63 @@ object GrayPacket { return best >= BACKGROUND_MIN_SHARE * count && bestStart + BACKGROUND_SPAN / 2 >= LIGHT_BACKGROUND_MIN_GRAY } + /** Pixels at least this opaque count as drawn, for inkBounds. */ + private const val INK_MIN_ALPHA = 16 + + /** + * The bounds of what translucent artwork draws (pixels at least INK_MIN_ALPHA opaque), as + * [left, top, right, bottom] with right and bottom exclusive; null when nothing is drawn. + * Used to trim a rendered layout's empty margins. + */ + @JvmStatic + fun inkBounds(pixels: IntArray, width: Int, height: Int): IntArray? { + var left = width + var top = height + var right = 0 + var bottom = 0 + for (y in 0 until height) { + val row = y * width + for (x in 0 until width) { + if (alpha(pixels[row + x]) >= INK_MIN_ALPHA) { + if (x < left) left = x + if (x >= right) right = x + 1 + if (y < top) top = y + bottom = y + 1 + } + } + } + return if (right > left && bottom > top) intArrayOf(left, top, right, bottom) else null + } + + /** + * Turns dark ink light, in place, for translucent artwork drawn with no background of its + * own (a notification's custom layout): returns whether it did. Ink whose average gray, + * weighted by alpha, is below mid-gray was drawn for a light background, like Dexcom's + * dark-gray text for a light-theme shade, and would vanish against the glasses' black; + * inverting its colors (not its alpha) shows it as it would look on white, as a negative. + * That is what invertLightBackground would make of the artwork composited on white, but + * it doesn't depend on the background covering enough of the picture to be detected. + */ + @JvmStatic + fun lightenDarkInk(pixels: IntArray, width: Int, height: Int): Boolean { + val count = width * height + var weight = 0L + var sum = 0L + for (i in 0 until count) { + val p = pixels[i] + val a = alpha(p) + weight += a + sum += luma(p).toLong() * a + } + if (weight == 0L || sum * 2 >= weight * 255) { + return false + } + for (i in 0 until count) { + pixels[i] = pixels[i] xor 0x00ffffff + } + return true + } + private fun luma(p: Int): Int { val r = (p shr 16) and 0xff val g = (p shr 8) and 0xff diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt index 3d228f5e..c2160361 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt @@ -3,6 +3,7 @@ package com.faceclaw.app import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals +import kotlin.test.assertNull import kotlin.test.assertSame import kotlin.test.assertTrue @@ -68,6 +69,38 @@ class GraphicsCoresTest { assertTrue(!GrayPacket.hasLightBackground(IntArray(8 * 8) { 0xff808080.toInt() }, 8, 8)) } + @Test + fun inkBoundsTrimTransparentMargins() { + val art = IntArray(8 * 6) + art[2 * 8 + 3] = 0xff000000.toInt() + art[4 * 8 + 5] = 0x80ffffff.toInt() + art[0] = 0x08ffffff // too faint to count + assertContentEquals(intArrayOf(3, 2, 6, 5), GrayPacket.inkBounds(art, 8, 6)) + assertNull(GrayPacket.inkBounds(IntArray(4 * 4), 4, 4)) + } + + @Test + fun darkInkOnTransparentTurnsLight() { + val darkGray = 0xff202020.toInt() + val text = IntArray(10 * 10) + for (i in 40 until 50) text[i] = darkGray + text[55] = 0x80202020.toInt() + assertTrue(GrayPacket.lightenDarkInk(text, 10, 10)) + assertEquals(0xffdfdfdf.toInt(), text[40]) + assertEquals(0x80dfdfdf.toInt(), text[55]) // alpha kept + assertEquals(0, text[0] ushr 24) // still transparent + // Light ink, an empty layout, and a light opaque panel are left alone (the panel is + // invertLightBackground's to handle). + val light = IntArray(10 * 10) + light[40] = 0xffe0e0e0.toInt() + assertTrue(!GrayPacket.lightenDarkInk(light, 10, 10)) + assertEquals(0xffe0e0e0.toInt(), light[40]) + assertTrue(!GrayPacket.lightenDarkInk(IntArray(4), 2, 2)) + val panel = IntArray(10 * 10) { 0xfff0f0f0.toInt() } + panel[5] = darkGray + assertTrue(!GrayPacket.lightenDarkInk(panel, 10, 10)) + } + @Test fun toneCurveDarkensMidtones() { val identity = GrayPacket.toneCurve(1f)