Center the phone screen preview

This commit is contained in:
James Babcock committed 2026-10-12 17:12:36 -07:00
1 parent f6201d8e41
commit 96e3bd95f7
18 files changed
+311 -24

No files matched your search

@@ -242,6 +242,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
} }
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
val composite = core.previewComposite() ?: return null
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
cropX, cropY, cropWidth, cropHeight)
}
/** /**
* Posted to the main looper after each step of an animation the preview is replaying (menu * Posted to the main looper after each step of an animation the preview is replaying (menu
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover * highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
@@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) {
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
} }
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
val composite = compositor.previewComposite() ?: return null
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
cropX, cropY, cropWidth, cropHeight)
}
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
@Throws(java.io.IOException::class) @Throws(java.io.IOException::class)
fun saveCompositePngScreenshot(): String { fun saveCompositePngScreenshot(): String {
@@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() {
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
} }
/**
* As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the
* mirror's Scaled preview). The area may extend past the frame's edges; it is black
* there.
*/
@JvmStatic
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw IllegalArgumentException("invalid gray preview buffer")
}
if (cropWidth <= 0 || cropHeight <= 0) {
throw IllegalArgumentException("invalid preview crop")
}
val bytes = ByteArray(width * height)
gray.get(bytes)
val cropped = ByteArray(cropWidth * cropHeight)
val left = maxOf(cropX, 0)
val right = minOf(cropX + cropWidth, width)
if (right > left) {
for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) {
System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left)
}
}
val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green)
return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
}
/** /**
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes * A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
* alpha; black, which the glasses show as unlit, is transparent), for * alpha; black, which the glasses show as unlit, is transparent), for
+13 -6
View File
@@ -46,6 +46,7 @@ const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteI
"swipe-right": "swipe-right", "swipe-right": "swipe-right",
}; };
import { GlanceHost } from "./glance-host"; import { GlanceHost } from "./glance-host";
import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop";
import { type GlanceEvent } from "./glance-state"; import { type GlanceEvent } from "./glance-state";
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress"; import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
@@ -69,7 +70,7 @@ import { ALL_APPS } from "../apps/all-apps";
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition"; import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry"; import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
@@ -273,6 +274,8 @@ class DashboardController {
private extractedFontsConfirmed = false; private extractedFontsConfirmed = false;
private screenRecordingActive = false; private screenRecordingActive = false;
private displayPreview: ImageSource | null = createInitialDisplayPreview(); private displayPreview: ImageSource | null = createInitialDisplayPreview();
/** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */
private displayPreviewCrop: ScreenRect | null = null;
private silentMode = false; private silentMode = false;
// Brightness value last pushed to the glasses; see pushBrightness. // Brightness value last pushed to the glasses; see pushBrightness.
private lastPushedBrightness: string | null = null; private lastPushedBrightness: string | null = null;
@@ -458,8 +461,9 @@ class DashboardController {
this.applyAppLayoutsIfChanged(); this.applyAppLayoutsIfChanged();
this.syncAssistantBridgeIfChanged(); this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged(); this.syncLockScreenSettingIfChanged();
// A Preview color change should show on the mirror at once, not at the // A Preview color or scaling change should show on the mirror at once,
// next composited frame (cheap: floor-limited, dedup'd on the Java side). // not at the next composited frame (cheap: floor-limited, dedup'd on
// the Java side).
this.schedulePreviewUpdate(); this.schedulePreviewUpdate();
}); });
// Connect to the external agent bridge at boot if configured; the // Connect to the external agent bridge at boot if configured; the
@@ -1931,8 +1935,9 @@ class DashboardController {
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch"); await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
return; return;
} }
const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH); const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny);
const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT); const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH);
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT);
const foreground = shell.foregroundWindow(); const foreground = shell.foregroundWindow();
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId); const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
this.appendLog(`mirror tap at ${x},${y}`); this.appendLog(`mirror tap at ${x},${y}`);
@@ -2883,8 +2888,10 @@ class DashboardController {
const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager; const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager;
if (!power?.isInteractive()) return; if (!power?.isInteractive()) return;
} }
const preview = display.getCompositePreview(previewColorSetting.get() === "green"); const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect());
const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop);
if (preview) { if (preview) {
this.displayPreviewCrop = crop;
this.setDisplayPreview(preview); this.setDisplayPreview(preview);
} }
} }
+9 -1
View File
@@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
import * as frameTimings from "../native/frame-timings"; import * as frameTimings from "../native/frame-timings";
import { type DisplayTarget } from "../native/preview-display"; import { type DisplayTarget } from "../native/preview-display";
import { minWindowTop } from "../ui/shell/geometry"; import { minWindowTop, type ScreenRect } from "../ui/shell/geometry";
import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition"; import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state"; import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
@@ -59,6 +59,8 @@ export class GlanceHost {
private rendering = false; private rendering = false;
private renderPending = false; private renderPending = false;
private nextRenderWantsFreshData = false; private nextRenderWantsFreshData = false;
/** Where the last render put the board on screen. */
private boardRect: ScreenRect | null = null;
constructor(private readonly options: GlanceHostOptions) {} constructor(private readonly options: GlanceHostOptions) {}
@@ -66,6 +68,11 @@ export class GlanceHost {
return this.state.visible; return this.state.visible;
} }
/** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */
screenRect(): ScreenRect | null {
return this.state.visible ? this.boardRect : null;
}
/** Whether sleep-time gestures should reach the board at all. */ /** Whether sleep-time gestures should reach the board at all. */
isEnabled(): boolean { isEnabled(): boolean {
return this.options.getProvider()?.isEnabled() ?? false; return this.options.getProvider()?.isEnabled() ?? false;
@@ -249,6 +256,7 @@ export class GlanceHost {
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0); const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop())); const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
boardImage.composeInto(full, x, y); boardImage.composeInto(full, x, y);
this.boardRect = { x, y, width: boardImage.width, height: boardImage.height };
return full; return full;
}), }),
); );
+9 -3
View File
@@ -43,7 +43,8 @@ import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/g
import { noopLayerActions, type LayerActions } from '../ui/layers' import { noopLayerActions, type LayerActions } from '../ui/layers'
import { TextViewerLayer } from '../apps/files/text-viewer' import { TextViewerLayer } from '../apps/files/text-viewer'
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry' import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry'
import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged, import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
@@ -116,6 +117,8 @@ export class IosPreviewController {
appendLog: message => this.logBluetooth(message), appendLog: message => this.logBluetooth(message),
}) })
private previewTimer: ReturnType<typeof setTimeout> | null = null private previewTimer: ReturnType<typeof setTimeout> | null = null
/** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */
private previewCrop: ScreenRect | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null private offSettings: (() => void) | null = null
private lastBrightness: string | null = null private lastBrightness: string | null = null
@@ -506,8 +509,10 @@ export class IosPreviewController {
this.previewTimer = null this.previewTimer = null
if (!this.active) return if (!this.active) return
try { try {
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green') const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect())
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop)
if (!image) return if (!image) return
this.previewCrop = crop
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) : `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
} catch (error) { this.fail(error) } } catch (error) { this.fail(error) }
@@ -561,7 +566,8 @@ export class IosPreviewController {
await shell.receiveInput(event) await shell.receiveInput(event)
this.requestShellRender() this.requestShellRender()
} }
private async mirrorTap(nx: number, ny: number): Promise<void> { private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise<void> {
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
const x = Math.max(0, Math.min(639, Math.floor(nx * 640))) const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480))) const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
const window = shell.foregroundWindow() const window = shell.foregroundWindow()
+33
View File
@@ -0,0 +1,33 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
import { previewScalingSetting } from "../ui/dashboard-settings";
import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry";
import { shell } from "../ui/shell/shell";
/**
* The screen area the phone's mirror shows (Settings > Phone display >
* Preview scaling), or null for the whole panel. Scaled zooms to the
* Glanceboard while it shows, else to the area the shell uses around the
* foreground window, centred at the panel's aspect. A locked display gets
* the whole panel: its message box is centred in the panel, not the band.
*/
export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null {
if (locked || previewScalingSetting.get() === "raw") return null;
const window = shell.foregroundWindow();
const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId));
const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT;
return wholePanel ? null : crop;
}
/**
* A point on the mirror, as fractions of the mirror's width and height,
* as fractions of the panel's, given the crop the mirror is showing.
* Points in the black margins past the panel's edges come out below 0 or
* above 1.
*/
export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } {
if (!crop) return { nx, ny };
return {
nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH,
ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT,
};
}
+3 -3
View File
@@ -489,13 +489,13 @@ export class FaceclawCommunicatorBridge {
}); });
} }
/** Phone-UI preview of the current composited screen, or null if none yet. */ /** Phone-UI preview of the current composited screen (or of `crop`'s area of it), or null if none yet. */
getCompositePreview(green = false): ImageSource | null { getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
const frame = this.communicator.previewComposite(); const frame = this.communicator.previewComposite();
if (!frame) return null; if (!frame) return null;
const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight()); const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight());
if (width <= 0 || height <= 0) return null; if (width <= 0 || height <= 0) return null;
return previewPixels(fromData(frame.pixels), width, height, green); return previewPixels(fromData(frame.pixels), width, height, green, crop);
} }
/** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */ /** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */
+13 -3
View File
@@ -507,11 +507,21 @@ export class FaceclawCommunicatorBridge {
/** /**
* Phone-UI preview of the current composited screen, or null if none yet. * Phone-UI preview of the current composited screen, or null if none yet.
* `green` tints it green-on-black (the Preview color setting) instead of * `green` tints it green-on-black (the Preview color setting) instead of
* grayscale. * grayscale; `crop` limits it to a screen area (the Preview scaling
* setting), black where that runs past the screen's edges.
*/ */
getCompositePreview(green = false): ImageSource | null { getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
if (!global.isAndroid) return null; if (!global.isAndroid) return null;
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); const bitmap = crop
? this.communicator.getCompositePreviewBitmap(
PREVIEW_BRIGHTEN_GAMMA,
green,
Math.round(crop.x),
Math.round(crop.y),
Math.round(crop.width),
Math.round(crop.height),
)
: this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
return bitmap ? new ImageSource(bitmap) : null; return bitmap ? new ImageSource(bitmap) : null;
} }
+17 -3
View File
@@ -3,11 +3,25 @@ import { GrayImage } from '../graphics/image'
declare const FaceclawGraphics: any declare const FaceclawGraphics: any
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource { type CropRect = { x: number; y: number; width: number; height: number }
/** The gray pixels of a screen area; black where it runs past the frame's edges. */
function cropGray(pixels: Uint8Array, width: number, height: number, crop: CropRect): Uint8Array {
const cropped = new Uint8Array(crop.width * crop.height)
const left = Math.max(crop.x, 0), right = Math.min(crop.x + crop.width, width)
if (right <= left) return cropped
for (let y = Math.max(crop.y, 0); y < Math.min(crop.y + crop.height, height); y++) {
cropped.set(pixels.subarray(y * width + left, y * width + right), (y - crop.y) * crop.width + (left - crop.x))
}
return cropped
}
/** Gray pixels as the phone mirror's image, optionally cropped to a screen area (Preview scaling). */
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false, crop: CropRect | null = null): ImageSource {
// NSData copies the buffer; the UIImage owns its pixels after this call. // NSData copies the buffer; the UIImage owns its pixels after this call.
const copy = new Uint8Array(pixels) const copy = crop ? cropGray(pixels, width, height, crop) : new Uint8Array(pixels)
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green) const native = FaceclawGraphics.previewWidthHeightGreen(data, crop?.width ?? width, crop?.height ?? height, green)
if (!native) throw new Error('Could not create preview image') if (!native) throw new Error('Could not create preview image')
return new ImageSource(native) return new ImageSource(native)
} }
+2 -2
View File
@@ -138,8 +138,8 @@ export class PreviewDisplayTarget implements DisplayTarget {
return this.compositor.composite(); return this.compositor.composite();
} }
getCompositePreview(green = false): ImageSource | null { getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green); return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green, crop);
} }
/** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */ /** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */
+11 -2
View File
@@ -160,9 +160,18 @@ export class PreviewDisplayTarget implements DisplayTarget {
return Promise.resolve("composited"); return Promise.resolve("composited");
} }
getCompositePreview(green = false): ImageSource | null { getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
if (!global.isAndroid) return null; if (!global.isAndroid) return null;
const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); const bitmap = crop
? this.compositor.getCompositePreviewBitmap(
PREVIEW_BRIGHTEN_GAMMA,
green,
Math.round(crop.x),
Math.round(crop.y),
Math.round(crop.width),
Math.round(crop.height),
)
: this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
return bitmap ? new ImageSource(bitmap) : null; return bitmap ? new ImageSource(bitmap) : null;
} }
+13
View File
@@ -616,6 +616,19 @@ export const previewColorSetting = new ConfigSettingEnum<PreviewColor>({
"How the phone's mirror of the glasses display renders: white/grayscale (clearest), or green to match the physical glasses.", "How the phone's mirror of the glasses display renders: white/grayscale (clearest), or green to match the physical glasses.",
}); });
export type PreviewScaling = "scaled" | "raw";
export const previewScalingSetting = new ConfigSettingEnum<PreviewScaling>({
id: "preview-scaling",
label: "Preview scaling",
storageKey: "phone.previewScaling",
defaultValue: "scaled",
values: ["scaled", "raw"],
formatValue: (value) => (value === "raw" ? "Raw" : "Scaled"),
description:
"How much of the glasses display the phone's mirror shows: Scaled centers and enlarges the part the current display mode uses; Raw shows the whole 640x480 panel.",
});
export const mirrorTouchSetting = new ConfigSettingBoolean({ export const mirrorTouchSetting = new ConfigSettingBoolean({
id: "mirror-touch", id: "mirror-touch",
label: "Touch mirror", label: "Touch mirror",
+2
View File
@@ -57,6 +57,7 @@ import {
mirrorTouchSetting, mirrorTouchSetting,
openAiApiKeySetting, openAiApiKeySetting,
previewColorSetting, previewColorSetting,
previewScalingSetting,
phoneRotationSetting, phoneRotationSetting,
ringConnectionModeSetting, ringConnectionModeSetting,
sonioxApiKeySetting, sonioxApiKeySetting,
@@ -162,6 +163,7 @@ function settingsSections(): SettingsSection[] {
items: [ items: [
enumSettingMenuItem(phoneRotationSetting), enumSettingMenuItem(phoneRotationSetting),
enumSettingMenuItem(previewColorSetting), enumSettingMenuItem(previewColorSetting),
enumSettingMenuItem(previewScalingSetting),
toggleSettingMenuItem(mirrorTouchSetting), toggleSettingMenuItem(mirrorTouchSetting),
], ],
}, },
+52
View File
@@ -422,3 +422,55 @@ export function appViewportRect(mode: WindowHeightMode, appId?: string): {
...appViewportSize(mode, appId), ...appViewportSize(mode, appId),
}; };
} }
export type ScreenRect = { x: number; y: number; width: number; height: number };
function unionRect(a: ScreenRect, b: ScreenRect): ScreenRect {
const x = Math.min(a.x, b.x);
const y = Math.min(a.y, b.y);
return {
x,
y,
width: Math.max(a.x + a.width, b.x + b.width) - x,
height: Math.max(a.y + a.height, b.y + b.height) - y,
};
}
/**
* The part of the screen the shell draws in while the given (foreground)
* window is up: the window's band, plus the frame's side pixels; the
* min-height band the shell overlays align to; and a side strip or bottom
* row (a popup shows within the window's content). An immersive window has
* the whole panel. The phone mirror's Scaled preview zooms to this.
*/
export function usedScreenRect(heightMode: WindowHeightMode, appId?: string): ScreenRect {
if (isImmersive(appId)) return { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT };
const frame = windowFramed(appId) ? 1 : 0;
const x = appViewportLeft(appId) - frame;
const width = appViewportWidth(appId) + 2 * frame;
let used = unionRect(
{ x, y: windowTop(heightMode, appId), width, height: windowBandHeight(heightMode, appId) },
{ x, y: minWindowTop(appId), width, height: MIN_WINDOW_HEIGHT },
);
if (switcherPosition() !== "popup") used = unionRect(used, switcherRect(heightMode, appId));
return used;
}
/**
* The screen area the phone mirror shows to present `used` as large as it
* fits, centred, at the panel's own 4:3 aspect (so the mirror keeps its
* shape whatever the display mode): `used` grown along one axis, past the
* panel's edges if need be (the mirror shows black there).
*/
export function phonePreviewRect(used: ScreenRect): ScreenRect {
// Whole 4:3 units, so the aspect is exact.
const units = Math.ceil(Math.max(used.width / 4, used.height / 3));
const width = units * 4;
const height = units * 3;
return {
x: Math.round(used.x + (used.width - width) / 2),
y: Math.round(used.y + (used.height - height) / 2),
width,
height,
};
}
+84
View File
@@ -263,6 +263,90 @@ test('an immersive app fills the panel without chrome rows, frame or depth, in a
} }
}); });
test('the phone mirror zooms to the area the shell uses, centred at the panel\'s 4:3', () => {
const g = geometry();
const preview = (mode, appId) => {
const used = { ...g.usedScreenRect(mode, appId) };
return { used, shown: { ...g.phonePreviewRect(used) } };
};
const panel = { x: 0, y: 0, width: 640, height: 480 };
// A side strip and band: the panel's width, so only centring vertically.
assert.deepEqual(preview('min'), { used: { x: 0, y: 96, width: 640, height: 288 }, shown: panel });
g.settings.displayLayoutSetting.value = 'band-top';
assert.deepEqual(preview('min').shown, { x: 0, y: -96, width: 640, height: 480 });
g.settings.appSwitcherPositionSetting.value = 'right';
g.settings.displayLayoutSetting.value = 'band-high';
assert.deepEqual(preview('min'), { used: { x: 0, y: 48, width: 640, height: 288 }, shown: { x: 0, y: -48, width: 640, height: 480 } });
// A bottom row: the band with the frame's sides, and the row under it.
g.settings.appSwitcherPositionSetting.value = 'bottom';
g.settings.displayLayoutSetting.value = 'band-middle';
assert.deepEqual(preview('min'), { used: { x: 31, y: 78, width: 578, height: 324 }, shown: { x: 30, y: 23, width: 580, height: 435 } });
// A tall window (the terminal's default) leaves just the frame's margins.
assert.deepEqual(preview('max', 'terminal'), { used: { x: 31, y: 0, width: 578, height: 480 }, shown: panel });
// A popup shows within the window: just the band.
g.settings.appSwitcherPositionSetting.value = 'popup';
assert.deepEqual(preview('min'), { used: { x: 31, y: 96, width: 578, height: 288 }, shown: { x: 30, y: 23, width: 580, height: 435 } });
// Full Screen and immersive windows use the whole panel.
g.settings.displayLayoutSetting.value = 'fullscreen';
assert.deepEqual(preview('min'), { used: panel, shown: panel });
g.settings.displayLayoutSetting.value = 'band-middle';
g.setAppImmersive('onboarding', true);
assert.deepEqual(preview('min', 'onboarding'), { used: panel, shown: panel });
// Every layout's shown area is exactly 4:3 and contains the used area.
for (const position of ['left', 'right', 'bottom', 'popup']) {
g.settings.appSwitcherPositionSetting.value = position;
for (const layout of LAYOUTS) {
g.settings.displayLayoutSetting.value = layout;
for (const mode of ['min', 'medium', 'max']) {
const { used, shown } = preview(mode);
assert.equal(shown.width * 3, shown.height * 4);
assert.ok(shown.width <= 640 && shown.height <= 480);
assert.ok(shown.x <= used.x && shown.y <= used.y);
assert.ok(shown.x + shown.width >= used.x + used.width && shown.y + shown.height >= used.y + used.height);
}
}
}
});
test('the mirror\'s crop follows Preview scaling, the Glanceboard and the lock, and maps taps back to the panel', () => {
const g = geometry();
const scaling = { value: 'scaled', get() { return this.value; } };
let foreground = { heightMode: 'min', appId: 'launcher' };
const modules = {
'../graphics/image': { G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 },
'../ui/dashboard-settings': { previewScalingSetting: scaling },
'../ui/shell/geometry': g,
'../ui/shell/shell': { shell: { foregroundWindow: () => foreground } },
};
const context = { exports: {}, require: (name) => modules[name] };
vm.runInNewContext(js(read('app/g2/phone-preview-crop.ts')), context);
const { phonePreviewCrop, mirrorToPanelFraction } = context.exports;
const crop = (locked = false, glance = null) => {
const rect = phonePreviewCrop(locked, glance);
return rect && { ...rect };
};
g.settings.appSwitcherPositionSetting.value = 'bottom';
const band = { x: 30, y: 23, width: 580, height: 435 };
assert.deepEqual(crop(), band);
// The whole panel is no crop at all.
foreground = { heightMode: 'max', appId: 'terminal' };
assert.equal(crop(), null);
foreground = null;
assert.deepEqual(crop(), band);
// The Glanceboard while it shows; a locked display's centred box needs the panel.
assert.deepEqual(crop(false, { x: 32, y: 48, width: 576, height: 432 }), { x: 32, y: 48, width: 576, height: 432 });
assert.deepEqual(crop(false, { x: 32, y: 96, width: 576, height: 288 }), { x: 32, y: 24, width: 576, height: 432 });
assert.equal(crop(true), null);
scaling.value = 'raw';
assert.equal(crop(), null);
// Taps: the mirror's corners are the crop's, its margins past the panel's edges.
assert.deepEqual({ ...mirrorToPanelFraction(null, 0.25, 0.75) }, { nx: 0.25, ny: 0.75 });
assert.deepEqual({ ...mirrorToPanelFraction(band, 0, 0) }, { nx: 30 / 640, ny: 23 / 480 });
assert.deepEqual({ ...mirrorToPanelFraction(band, 1, 1) }, { nx: 610 / 640, ny: 458 / 480 });
const above = mirrorToPanelFraction({ x: 0, y: -96, width: 640, height: 480 }, 0.5, 0.1);
assert.ok(above.ny < 0);
});
// The real chrome layer over the given geometry, with platform pieces stubbed: // The real chrome layer over the given geometry, with platform pieces stubbed:
// a 4px-per-character font that paints each character as a 2x6 block, up to // a 4px-per-character font that paints each character as a 2x6 block, up to
// `notifications` 24px notification icons (keyed n0, n1, ...), and a phone // `notifications` 24px notification icons (keyed n0, n1, ...), and a phone
+2
View File
@@ -145,6 +145,8 @@ function fixture() {
'../native/nightscout-bridge': { nightscoutBridge: { async start() {}, async stop() {} } }, '../native/nightscout-bridge': { nightscoutBridge: { async start() {}, async stop() {} } },
'../native/weather': { weatherBridge: { setSessionActive() {} } }, '../native/weather': { weatherBridge: { setSessionActive() {} } },
'./glance-host': { GlanceHost }, './events': events, './glance-host': { GlanceHost }, './events': events,
// Whole-panel mirror: the pixel assertions below index the full 640x480 preview.
'./phone-preview-crop': { phonePreviewCrop: () => null, mirrorToPanelFraction: (_crop, nx, ny) => ({ nx, ny }) },
'./lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) }, './lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) },
'./glasses-presence': load('app/g2/glasses-presence.ts', {}), './glasses-presence': load('app/g2/glasses-presence.ts', {}),
'./device-addresses': { loadDeviceAddresses: () => ({ right: 'AA', left: 'BB', ring: '' }) }, './ios-peripheral-identity': { deviceAddressError: () => null }, './device-addresses': { loadDeviceAddresses: () => ({ right: 'AA', left: 'BB', ring: '' }) }, './ios-peripheral-identity': { deviceAddressError: () => null },
+2 -1
View File
@@ -17,7 +17,7 @@ function harness() {
Application: { android: { get foregroundActivity() { return activityPresent ? activity : null; } } }, Application: { android: { get foregroundActivity() { return activityPresent ? activity : null; } } },
android: { content: { Context: { POWER_SERVICE: 'power' } } }, android: { content: { Context: { POWER_SERVICE: 'power' } } },
Date: { now: () => now }, CONNECTED_PREVIEW_MIN_UPDATE_MS: 100, RECORDING_MIN_CAPTURE_MS: 200, Date: { now: () => now }, CONNECTED_PREVIEW_MIN_UPDATE_MS: 100, RECORDING_MIN_CAPTURE_MS: 200,
previewColorSetting: { get: () => 'green' }, previewColorSetting: { get: () => 'green' }, phonePreviewCrop: () => null,
setTimeout: fn => { const id = timers.size + 1; timers.set(id, fn); return id; }, setTimeout: fn => { const id = timers.size + 1; timers.set(id, fn); return id; },
clearTimeout: id => timers.delete(id), clearTimeout: id => timers.delete(id),
}; };
@@ -27,6 +27,7 @@ function harness() {
const instance = new context.exports.Harness(); const instance = new context.exports.Harness();
Object.assign(instance, { lastConnectedPreviewUpdateAtMs: 0, lastRecordCaptureAtMs: 0, Object.assign(instance, { lastConnectedPreviewUpdateAtMs: 0, lastRecordCaptureAtMs: 0,
phonePreviewVisible: null, previewTrailingTimer: null, screenRecordingActive: false, phonePreviewVisible: null, previewTrailingTimer: null, screenRecordingActive: false,
glassesLocked: false, glance: { screenRect: () => null },
display: { getCompositePreview: () => { previews++; return {}; }, recordScreenFrame: () => records++ }, display: { getCompositePreview: () => { previews++; return {}; }, recordScreenFrame: () => records++ },
setDisplayPreview() {}, setDisplayPreview() {},
}); });