forked from Blendman974/faceclaw
Merge branch 'main' into nightscout-missed-bolus-detector
This commit is contained in:
commit
8b1b914ee7
16 files changed
+307
-23
No files matched your search
@@ -242,6 +242,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
|||||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||||
|
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||||
|
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||||
|
val composite = core.previewComposite() ?: return null
|
||||||
|
return PreviewBitmapUtil.fromGray(
|
||||||
|
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||||
|
cropX, cropY, cropWidth, cropHeight)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Posted to the main looper after each step of an animation the preview is replaying (menu
|
* Posted to the main looper after each step of an animation the preview is replaying (menu
|
||||||
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
|
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
|
||||||
|
|||||||
@@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) {
|
|||||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||||
|
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||||
|
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||||
|
val composite = compositor.previewComposite() ?: return null
|
||||||
|
return PreviewBitmapUtil.fromGray(
|
||||||
|
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||||
|
cropX, cropY, cropWidth, cropHeight)
|
||||||
|
}
|
||||||
|
|
||||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||||
@Throws(java.io.IOException::class)
|
@Throws(java.io.IOException::class)
|
||||||
fun saveCompositePngScreenshot(): String {
|
fun saveCompositePngScreenshot(): String {
|
||||||
|
|||||||
@@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() {
|
|||||||
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
|
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the
|
||||||
|
* mirror's Scaled preview). The area may extend past the frame's edges; it is black
|
||||||
|
* there.
|
||||||
|
*/
|
||||||
|
@JvmStatic
|
||||||
|
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean,
|
||||||
|
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap {
|
||||||
|
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||||
|
throw IllegalArgumentException("invalid gray preview buffer")
|
||||||
|
}
|
||||||
|
if (cropWidth <= 0 || cropHeight <= 0) {
|
||||||
|
throw IllegalArgumentException("invalid preview crop")
|
||||||
|
}
|
||||||
|
val bytes = ByteArray(width * height)
|
||||||
|
gray.get(bytes)
|
||||||
|
val cropped = ByteArray(cropWidth * cropHeight)
|
||||||
|
val left = maxOf(cropX, 0)
|
||||||
|
val right = minOf(cropX + cropWidth, width)
|
||||||
|
if (right > left) {
|
||||||
|
for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) {
|
||||||
|
System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green)
|
||||||
|
return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
|
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
|
||||||
* alpha; black, which the glasses show as unlit, is transparent), for
|
* alpha; black, which the glasses show as unlit, is transparent), for
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteI
|
|||||||
"swipe-right": "swipe-right",
|
"swipe-right": "swipe-right",
|
||||||
};
|
};
|
||||||
import { GlanceHost } from "./glance-host";
|
import { GlanceHost } from "./glance-host";
|
||||||
|
import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop";
|
||||||
import { type GlanceEvent } from "./glance-state";
|
import { type GlanceEvent } from "./glance-state";
|
||||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
||||||
@@ -69,7 +70,7 @@ import { ALL_APPS } from "../apps/all-apps";
|
|||||||
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
|
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
|
||||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry";
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||||
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||||
@@ -273,6 +274,8 @@ class DashboardController {
|
|||||||
private extractedFontsConfirmed = false;
|
private extractedFontsConfirmed = false;
|
||||||
private screenRecordingActive = false;
|
private screenRecordingActive = false;
|
||||||
private displayPreview: ImageSource | null = createInitialDisplayPreview();
|
private displayPreview: ImageSource | null = createInitialDisplayPreview();
|
||||||
|
/** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */
|
||||||
|
private displayPreviewCrop: ScreenRect | null = null;
|
||||||
private silentMode = false;
|
private silentMode = false;
|
||||||
// Brightness value last pushed to the glasses; see pushBrightness.
|
// Brightness value last pushed to the glasses; see pushBrightness.
|
||||||
private lastPushedBrightness: string | null = null;
|
private lastPushedBrightness: string | null = null;
|
||||||
@@ -458,8 +461,9 @@ class DashboardController {
|
|||||||
this.applyAppLayoutsIfChanged();
|
this.applyAppLayoutsIfChanged();
|
||||||
this.syncAssistantBridgeIfChanged();
|
this.syncAssistantBridgeIfChanged();
|
||||||
this.syncLockScreenSettingIfChanged();
|
this.syncLockScreenSettingIfChanged();
|
||||||
// A Preview color change should show on the mirror at once, not at the
|
// A Preview color or scaling change should show on the mirror at once,
|
||||||
// next composited frame (cheap: floor-limited, dedup'd on the Java side).
|
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||||
|
// the Java side).
|
||||||
this.schedulePreviewUpdate();
|
this.schedulePreviewUpdate();
|
||||||
});
|
});
|
||||||
// Connect to the external agent bridge at boot if configured; the
|
// Connect to the external agent bridge at boot if configured; the
|
||||||
@@ -1931,8 +1935,9 @@ class DashboardController {
|
|||||||
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
|
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH);
|
const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny);
|
||||||
const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT);
|
const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH);
|
||||||
|
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT);
|
||||||
const foreground = shell.foregroundWindow();
|
const foreground = shell.foregroundWindow();
|
||||||
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
|
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
|
||||||
this.appendLog(`mirror tap at ${x},${y}`);
|
this.appendLog(`mirror tap at ${x},${y}`);
|
||||||
@@ -2883,8 +2888,10 @@ class DashboardController {
|
|||||||
const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager;
|
const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager;
|
||||||
if (!power?.isInteractive()) return;
|
if (!power?.isInteractive()) return;
|
||||||
}
|
}
|
||||||
const preview = display.getCompositePreview(previewColorSetting.get() === "green");
|
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect());
|
||||||
|
const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop);
|
||||||
if (preview) {
|
if (preview) {
|
||||||
|
this.displayPreviewCrop = crop;
|
||||||
this.setDisplayPreview(preview);
|
this.setDisplayPreview(preview);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire";
|
|||||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||||
import * as frameTimings from "../native/frame-timings";
|
import * as frameTimings from "../native/frame-timings";
|
||||||
import { type DisplayTarget } from "../native/preview-display";
|
import { type DisplayTarget } from "../native/preview-display";
|
||||||
import { minWindowTop } from "../ui/shell/geometry";
|
import { minWindowTop, type ScreenRect } from "../ui/shell/geometry";
|
||||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||||
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
|
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
|
||||||
import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
|
import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
|
||||||
@@ -59,6 +59,8 @@ export class GlanceHost {
|
|||||||
private rendering = false;
|
private rendering = false;
|
||||||
private renderPending = false;
|
private renderPending = false;
|
||||||
private nextRenderWantsFreshData = false;
|
private nextRenderWantsFreshData = false;
|
||||||
|
/** Where the last render put the board on screen. */
|
||||||
|
private boardRect: ScreenRect | null = null;
|
||||||
|
|
||||||
constructor(private readonly options: GlanceHostOptions) {}
|
constructor(private readonly options: GlanceHostOptions) {}
|
||||||
|
|
||||||
@@ -66,6 +68,11 @@ export class GlanceHost {
|
|||||||
return this.state.visible;
|
return this.state.visible;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */
|
||||||
|
screenRect(): ScreenRect | null {
|
||||||
|
return this.state.visible ? this.boardRect : null;
|
||||||
|
}
|
||||||
|
|
||||||
/** Whether sleep-time gestures should reach the board at all. */
|
/** Whether sleep-time gestures should reach the board at all. */
|
||||||
isEnabled(): boolean {
|
isEnabled(): boolean {
|
||||||
return this.options.getProvider()?.isEnabled() ?? false;
|
return this.options.getProvider()?.isEnabled() ?? false;
|
||||||
@@ -249,6 +256,7 @@ export class GlanceHost {
|
|||||||
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
|
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
|
||||||
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
|
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
|
||||||
boardImage.composeInto(full, x, y);
|
boardImage.composeInto(full, x, y);
|
||||||
|
this.boardRect = { x, y, width: boardImage.width, height: boardImage.height };
|
||||||
return full;
|
return full;
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -43,7 +43,8 @@ import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/g
|
|||||||
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||||
import { TextViewerLayer } from '../apps/files/text-viewer'
|
import { TextViewerLayer } from '../apps/files/text-viewer'
|
||||||
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
||||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry'
|
||||||
|
import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||||
import { onAnySettingChanged,
|
import { onAnySettingChanged,
|
||||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||||
@@ -116,6 +117,8 @@ export class IosPreviewController {
|
|||||||
appendLog: message => this.logBluetooth(message),
|
appendLog: message => this.logBluetooth(message),
|
||||||
})
|
})
|
||||||
private previewTimer: ReturnType<typeof setTimeout> | null = null
|
private previewTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
|
/** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */
|
||||||
|
private previewCrop: ScreenRect | null = null
|
||||||
private clockTimer: ReturnType<typeof setInterval> | null = null
|
private clockTimer: ReturnType<typeof setInterval> | null = null
|
||||||
private offSettings: (() => void) | null = null
|
private offSettings: (() => void) | null = null
|
||||||
private lastBrightness: string | null = null
|
private lastBrightness: string | null = null
|
||||||
@@ -506,8 +509,10 @@ export class IosPreviewController {
|
|||||||
this.previewTimer = null
|
this.previewTimer = null
|
||||||
if (!this.active) return
|
if (!this.active) return
|
||||||
try {
|
try {
|
||||||
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green')
|
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect())
|
||||||
|
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop)
|
||||||
if (!image) return
|
if (!image) return
|
||||||
|
this.previewCrop = crop
|
||||||
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
||||||
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||||
} catch (error) { this.fail(error) }
|
} catch (error) { this.fail(error) }
|
||||||
@@ -561,7 +566,8 @@ export class IosPreviewController {
|
|||||||
await shell.receiveInput(event)
|
await shell.receiveInput(event)
|
||||||
this.requestShellRender()
|
this.requestShellRender()
|
||||||
}
|
}
|
||||||
private async mirrorTap(nx: number, ny: number): Promise<void> {
|
private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise<void> {
|
||||||
|
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
|
||||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||||
const window = shell.foregroundWindow()
|
const window = shell.foregroundWindow()
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
|
||||||
|
import { previewScalingSetting } from "../ui/dashboard-settings";
|
||||||
|
import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry";
|
||||||
|
import { shell } from "../ui/shell/shell";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The screen area the phone's mirror shows (Settings > Phone display >
|
||||||
|
* Preview scaling), or null for the whole panel. Scaled zooms to the
|
||||||
|
* Glanceboard while it shows, else to the area the shell uses around the
|
||||||
|
* foreground window, centred at the panel's aspect. A locked display gets
|
||||||
|
* the whole panel: its message box is centred in the panel, not the band.
|
||||||
|
*/
|
||||||
|
export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null {
|
||||||
|
if (locked || previewScalingSetting.get() === "raw") return null;
|
||||||
|
const window = shell.foregroundWindow();
|
||||||
|
const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId));
|
||||||
|
const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT;
|
||||||
|
return wholePanel ? null : crop;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A point on the mirror, as fractions of the mirror's width and height,
|
||||||
|
* as fractions of the panel's, given the crop the mirror is showing.
|
||||||
|
* Points in the black margins past the panel's edges come out below 0 or
|
||||||
|
* above 1.
|
||||||
|
*/
|
||||||
|
export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } {
|
||||||
|
if (!crop) return { nx, ny };
|
||||||
|
return {
|
||||||
|
nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH,
|
||||||
|
ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -489,13 +489,13 @@ export class FaceclawCommunicatorBridge {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Phone-UI preview of the current composited screen, or null if none yet. */
|
/** Phone-UI preview of the current composited screen (or of `crop`'s area of it), or null if none yet. */
|
||||||
getCompositePreview(green = false): ImageSource | null {
|
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
|
||||||
const frame = this.communicator.previewComposite();
|
const frame = this.communicator.previewComposite();
|
||||||
if (!frame) return null;
|
if (!frame) return null;
|
||||||
const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight());
|
const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight());
|
||||||
if (width <= 0 || height <= 0) return null;
|
if (width <= 0 || height <= 0) return null;
|
||||||
return previewPixels(fromData(frame.pixels), width, height, green);
|
return previewPixels(fromData(frame.pixels), width, height, green, crop);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */
|
/** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */
|
||||||
|
|||||||
@@ -507,11 +507,21 @@ export class FaceclawCommunicatorBridge {
|
|||||||
/**
|
/**
|
||||||
* Phone-UI preview of the current composited screen, or null if none yet.
|
* Phone-UI preview of the current composited screen, or null if none yet.
|
||||||
* `green` tints it green-on-black (the Preview color setting) instead of
|
* `green` tints it green-on-black (the Preview color setting) instead of
|
||||||
* grayscale.
|
* grayscale; `crop` limits it to a screen area (the Preview scaling
|
||||||
|
* setting), black where that runs past the screen's edges.
|
||||||
*/
|
*/
|
||||||
getCompositePreview(green = false): ImageSource | null {
|
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
|
||||||
if (!global.isAndroid) return null;
|
if (!global.isAndroid) return null;
|
||||||
const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
const bitmap = crop
|
||||||
|
? this.communicator.getCompositePreviewBitmap(
|
||||||
|
PREVIEW_BRIGHTEN_GAMMA,
|
||||||
|
green,
|
||||||
|
Math.round(crop.x),
|
||||||
|
Math.round(crop.y),
|
||||||
|
Math.round(crop.width),
|
||||||
|
Math.round(crop.height),
|
||||||
|
)
|
||||||
|
: this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
||||||
return bitmap ? new ImageSource(bitmap) : null;
|
return bitmap ? new ImageSource(bitmap) : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,11 +3,25 @@ import { GrayImage } from '../graphics/image'
|
|||||||
|
|
||||||
declare const FaceclawGraphics: any
|
declare const FaceclawGraphics: any
|
||||||
|
|
||||||
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
|
type CropRect = { x: number; y: number; width: number; height: number }
|
||||||
|
|
||||||
|
/** The gray pixels of a screen area; black where it runs past the frame's edges. */
|
||||||
|
function cropGray(pixels: Uint8Array, width: number, height: number, crop: CropRect): Uint8Array {
|
||||||
|
const cropped = new Uint8Array(crop.width * crop.height)
|
||||||
|
const left = Math.max(crop.x, 0), right = Math.min(crop.x + crop.width, width)
|
||||||
|
if (right <= left) return cropped
|
||||||
|
for (let y = Math.max(crop.y, 0); y < Math.min(crop.y + crop.height, height); y++) {
|
||||||
|
cropped.set(pixels.subarray(y * width + left, y * width + right), (y - crop.y) * crop.width + (left - crop.x))
|
||||||
|
}
|
||||||
|
return cropped
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Gray pixels as the phone mirror's image, optionally cropped to a screen area (Preview scaling). */
|
||||||
|
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false, crop: CropRect | null = null): ImageSource {
|
||||||
// NSData copies the buffer; the UIImage owns its pixels after this call.
|
// NSData copies the buffer; the UIImage owns its pixels after this call.
|
||||||
const copy = new Uint8Array(pixels)
|
const copy = crop ? cropGray(pixels, width, height, crop) : new Uint8Array(pixels)
|
||||||
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||||
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
|
const native = FaceclawGraphics.previewWidthHeightGreen(data, crop?.width ?? width, crop?.height ?? height, green)
|
||||||
if (!native) throw new Error('Could not create preview image')
|
if (!native) throw new Error('Could not create preview image')
|
||||||
return new ImageSource(native)
|
return new ImageSource(native)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -138,8 +138,8 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
|||||||
return this.compositor.composite();
|
return this.compositor.composite();
|
||||||
}
|
}
|
||||||
|
|
||||||
getCompositePreview(green = false): ImageSource | null {
|
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
|
||||||
return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green);
|
return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green, crop);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */
|
/** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */
|
||||||
|
|||||||
@@ -160,9 +160,18 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
|||||||
return Promise.resolve("composited");
|
return Promise.resolve("composited");
|
||||||
}
|
}
|
||||||
|
|
||||||
getCompositePreview(green = false): ImageSource | null {
|
getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null {
|
||||||
if (!global.isAndroid) return null;
|
if (!global.isAndroid) return null;
|
||||||
const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
const bitmap = crop
|
||||||
|
? this.compositor.getCompositePreviewBitmap(
|
||||||
|
PREVIEW_BRIGHTEN_GAMMA,
|
||||||
|
green,
|
||||||
|
Math.round(crop.x),
|
||||||
|
Math.round(crop.y),
|
||||||
|
Math.round(crop.width),
|
||||||
|
Math.round(crop.height),
|
||||||
|
)
|
||||||
|
: this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
|
||||||
return bitmap ? new ImageSource(bitmap) : null;
|
return bitmap ? new ImageSource(bitmap) : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -616,6 +616,19 @@ export const previewColorSetting = new ConfigSettingEnum<PreviewColor>({
|
|||||||
"How the phone's mirror of the glasses display renders: white/grayscale (clearest), or green to match the physical glasses.",
|
"How the phone's mirror of the glasses display renders: white/grayscale (clearest), or green to match the physical glasses.",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export type PreviewScaling = "scaled" | "raw";
|
||||||
|
|
||||||
|
export const previewScalingSetting = new ConfigSettingEnum<PreviewScaling>({
|
||||||
|
id: "preview-scaling",
|
||||||
|
label: "Preview scaling",
|
||||||
|
storageKey: "phone.previewScaling",
|
||||||
|
defaultValue: "scaled",
|
||||||
|
values: ["scaled", "raw"],
|
||||||
|
formatValue: (value) => (value === "raw" ? "Raw" : "Scaled"),
|
||||||
|
description:
|
||||||
|
"How much of the glasses display the phone's mirror shows: Scaled centers and enlarges the part the current display mode uses; Raw shows the whole 640x480 panel.",
|
||||||
|
});
|
||||||
|
|
||||||
export const mirrorTouchSetting = new ConfigSettingBoolean({
|
export const mirrorTouchSetting = new ConfigSettingBoolean({
|
||||||
id: "mirror-touch",
|
id: "mirror-touch",
|
||||||
label: "Touch mirror",
|
label: "Touch mirror",
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ import {
|
|||||||
mirrorTouchSetting,
|
mirrorTouchSetting,
|
||||||
openAiApiKeySetting,
|
openAiApiKeySetting,
|
||||||
previewColorSetting,
|
previewColorSetting,
|
||||||
|
previewScalingSetting,
|
||||||
phoneRotationSetting,
|
phoneRotationSetting,
|
||||||
ringConnectionModeSetting,
|
ringConnectionModeSetting,
|
||||||
sonioxApiKeySetting,
|
sonioxApiKeySetting,
|
||||||
@@ -162,6 +163,7 @@ function settingsSections(): SettingsSection[] {
|
|||||||
items: [
|
items: [
|
||||||
enumSettingMenuItem(phoneRotationSetting),
|
enumSettingMenuItem(phoneRotationSetting),
|
||||||
enumSettingMenuItem(previewColorSetting),
|
enumSettingMenuItem(previewColorSetting),
|
||||||
|
enumSettingMenuItem(previewScalingSetting),
|
||||||
toggleSettingMenuItem(mirrorTouchSetting),
|
toggleSettingMenuItem(mirrorTouchSetting),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -422,3 +422,55 @@ export function appViewportRect(mode: WindowHeightMode, appId?: string): {
|
|||||||
...appViewportSize(mode, appId),
|
...appViewportSize(mode, appId),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type ScreenRect = { x: number; y: number; width: number; height: number };
|
||||||
|
|
||||||
|
function unionRect(a: ScreenRect, b: ScreenRect): ScreenRect {
|
||||||
|
const x = Math.min(a.x, b.x);
|
||||||
|
const y = Math.min(a.y, b.y);
|
||||||
|
return {
|
||||||
|
x,
|
||||||
|
y,
|
||||||
|
width: Math.max(a.x + a.width, b.x + b.width) - x,
|
||||||
|
height: Math.max(a.y + a.height, b.y + b.height) - y,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The part of the screen the shell draws in while the given (foreground)
|
||||||
|
* window is up: the window's band, plus the frame's side pixels; the
|
||||||
|
* min-height band the shell overlays align to; and a side strip or bottom
|
||||||
|
* row (a popup shows within the window's content). An immersive window has
|
||||||
|
* the whole panel. The phone mirror's Scaled preview zooms to this.
|
||||||
|
*/
|
||||||
|
export function usedScreenRect(heightMode: WindowHeightMode, appId?: string): ScreenRect {
|
||||||
|
if (isImmersive(appId)) return { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT };
|
||||||
|
const frame = windowFramed(appId) ? 1 : 0;
|
||||||
|
const x = appViewportLeft(appId) - frame;
|
||||||
|
const width = appViewportWidth(appId) + 2 * frame;
|
||||||
|
let used = unionRect(
|
||||||
|
{ x, y: windowTop(heightMode, appId), width, height: windowBandHeight(heightMode, appId) },
|
||||||
|
{ x, y: minWindowTop(appId), width, height: MIN_WINDOW_HEIGHT },
|
||||||
|
);
|
||||||
|
if (switcherPosition() !== "popup") used = unionRect(used, switcherRect(heightMode, appId));
|
||||||
|
return used;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The screen area the phone mirror shows to present `used` as large as it
|
||||||
|
* fits, centred, at the panel's own 4:3 aspect (so the mirror keeps its
|
||||||
|
* shape whatever the display mode): `used` grown along one axis, past the
|
||||||
|
* panel's edges if need be (the mirror shows black there).
|
||||||
|
*/
|
||||||
|
export function phonePreviewRect(used: ScreenRect): ScreenRect {
|
||||||
|
// Whole 4:3 units, so the aspect is exact.
|
||||||
|
const units = Math.ceil(Math.max(used.width / 4, used.height / 3));
|
||||||
|
const width = units * 4;
|
||||||
|
const height = units * 3;
|
||||||
|
return {
|
||||||
|
x: Math.round(used.x + (used.width - width) / 2),
|
||||||
|
y: Math.round(used.y + (used.height - height) / 2),
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -263,6 +263,90 @@ test('an immersive app fills the panel without chrome rows, frame or depth, in a
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('the phone mirror zooms to the area the shell uses, centred at the panel\'s 4:3', () => {
|
||||||
|
const g = geometry();
|
||||||
|
const preview = (mode, appId) => {
|
||||||
|
const used = { ...g.usedScreenRect(mode, appId) };
|
||||||
|
return { used, shown: { ...g.phonePreviewRect(used) } };
|
||||||
|
};
|
||||||
|
const panel = { x: 0, y: 0, width: 640, height: 480 };
|
||||||
|
// A side strip and band: the panel's width, so only centring vertically.
|
||||||
|
assert.deepEqual(preview('min'), { used: { x: 0, y: 96, width: 640, height: 288 }, shown: panel });
|
||||||
|
g.settings.displayLayoutSetting.value = 'band-top';
|
||||||
|
assert.deepEqual(preview('min').shown, { x: 0, y: -96, width: 640, height: 480 });
|
||||||
|
g.settings.appSwitcherPositionSetting.value = 'right';
|
||||||
|
g.settings.displayLayoutSetting.value = 'band-high';
|
||||||
|
assert.deepEqual(preview('min'), { used: { x: 0, y: 48, width: 640, height: 288 }, shown: { x: 0, y: -48, width: 640, height: 480 } });
|
||||||
|
// A bottom row: the band with the frame's sides, and the row under it.
|
||||||
|
g.settings.appSwitcherPositionSetting.value = 'bottom';
|
||||||
|
g.settings.displayLayoutSetting.value = 'band-middle';
|
||||||
|
assert.deepEqual(preview('min'), { used: { x: 31, y: 78, width: 578, height: 324 }, shown: { x: 30, y: 23, width: 580, height: 435 } });
|
||||||
|
// A tall window (the terminal's default) leaves just the frame's margins.
|
||||||
|
assert.deepEqual(preview('max', 'terminal'), { used: { x: 31, y: 0, width: 578, height: 480 }, shown: panel });
|
||||||
|
// A popup shows within the window: just the band.
|
||||||
|
g.settings.appSwitcherPositionSetting.value = 'popup';
|
||||||
|
assert.deepEqual(preview('min'), { used: { x: 31, y: 96, width: 578, height: 288 }, shown: { x: 30, y: 23, width: 580, height: 435 } });
|
||||||
|
// Full Screen and immersive windows use the whole panel.
|
||||||
|
g.settings.displayLayoutSetting.value = 'fullscreen';
|
||||||
|
assert.deepEqual(preview('min'), { used: panel, shown: panel });
|
||||||
|
g.settings.displayLayoutSetting.value = 'band-middle';
|
||||||
|
g.setAppImmersive('onboarding', true);
|
||||||
|
assert.deepEqual(preview('min', 'onboarding'), { used: panel, shown: panel });
|
||||||
|
// Every layout's shown area is exactly 4:3 and contains the used area.
|
||||||
|
for (const position of ['left', 'right', 'bottom', 'popup']) {
|
||||||
|
g.settings.appSwitcherPositionSetting.value = position;
|
||||||
|
for (const layout of LAYOUTS) {
|
||||||
|
g.settings.displayLayoutSetting.value = layout;
|
||||||
|
for (const mode of ['min', 'medium', 'max']) {
|
||||||
|
const { used, shown } = preview(mode);
|
||||||
|
assert.equal(shown.width * 3, shown.height * 4);
|
||||||
|
assert.ok(shown.width <= 640 && shown.height <= 480);
|
||||||
|
assert.ok(shown.x <= used.x && shown.y <= used.y);
|
||||||
|
assert.ok(shown.x + shown.width >= used.x + used.width && shown.y + shown.height >= used.y + used.height);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the mirror\'s crop follows Preview scaling, the Glanceboard and the lock, and maps taps back to the panel', () => {
|
||||||
|
const g = geometry();
|
||||||
|
const scaling = { value: 'scaled', get() { return this.value; } };
|
||||||
|
let foreground = { heightMode: 'min', appId: 'launcher' };
|
||||||
|
const modules = {
|
||||||
|
'../graphics/image': { G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 },
|
||||||
|
'../ui/dashboard-settings': { previewScalingSetting: scaling },
|
||||||
|
'../ui/shell/geometry': g,
|
||||||
|
'../ui/shell/shell': { shell: { foregroundWindow: () => foreground } },
|
||||||
|
};
|
||||||
|
const context = { exports: {}, require: (name) => modules[name] };
|
||||||
|
vm.runInNewContext(js(read('app/g2/phone-preview-crop.ts')), context);
|
||||||
|
const { phonePreviewCrop, mirrorToPanelFraction } = context.exports;
|
||||||
|
const crop = (locked = false, glance = null) => {
|
||||||
|
const rect = phonePreviewCrop(locked, glance);
|
||||||
|
return rect && { ...rect };
|
||||||
|
};
|
||||||
|
g.settings.appSwitcherPositionSetting.value = 'bottom';
|
||||||
|
const band = { x: 30, y: 23, width: 580, height: 435 };
|
||||||
|
assert.deepEqual(crop(), band);
|
||||||
|
// The whole panel is no crop at all.
|
||||||
|
foreground = { heightMode: 'max', appId: 'terminal' };
|
||||||
|
assert.equal(crop(), null);
|
||||||
|
foreground = null;
|
||||||
|
assert.deepEqual(crop(), band);
|
||||||
|
// The Glanceboard while it shows; a locked display's centred box needs the panel.
|
||||||
|
assert.deepEqual(crop(false, { x: 32, y: 48, width: 576, height: 432 }), { x: 32, y: 48, width: 576, height: 432 });
|
||||||
|
assert.deepEqual(crop(false, { x: 32, y: 96, width: 576, height: 288 }), { x: 32, y: 24, width: 576, height: 432 });
|
||||||
|
assert.equal(crop(true), null);
|
||||||
|
scaling.value = 'raw';
|
||||||
|
assert.equal(crop(), null);
|
||||||
|
// Taps: the mirror's corners are the crop's, its margins past the panel's edges.
|
||||||
|
assert.deepEqual({ ...mirrorToPanelFraction(null, 0.25, 0.75) }, { nx: 0.25, ny: 0.75 });
|
||||||
|
assert.deepEqual({ ...mirrorToPanelFraction(band, 0, 0) }, { nx: 30 / 640, ny: 23 / 480 });
|
||||||
|
assert.deepEqual({ ...mirrorToPanelFraction(band, 1, 1) }, { nx: 610 / 640, ny: 458 / 480 });
|
||||||
|
const above = mirrorToPanelFraction({ x: 0, y: -96, width: 640, height: 480 }, 0.5, 0.1);
|
||||||
|
assert.ok(above.ny < 0);
|
||||||
|
});
|
||||||
|
|
||||||
// The real chrome layer over the given geometry, with platform pieces stubbed:
|
// The real chrome layer over the given geometry, with platform pieces stubbed:
|
||||||
// a 4px-per-character font that paints each character as a 2x6 block, up to
|
// a 4px-per-character font that paints each character as a 2x6 block, up to
|
||||||
// `notifications` 24px notification icons (keyed n0, n1, ...), and a phone
|
// `notifications` 24px notification icons (keyed n0, n1, ...), and a phone
|
||||||
|
|||||||
Reference in new issue
Block a user