Add screen-recording option

This commit is contained in:
James Babcock committed 2026-07-28 10:16:44 -07:00
1 parent 4c946631e0
commit b738ce4bb2
12 files changed
+644 -99

No files matched your search

+58 -9
View File
@@ -90,7 +90,6 @@ import {
firmwareDebugFlagsSetting,
nightscoutSiteUrlSetting,
onAnySettingChanged,
rawScreenshotsEnabledSetting,
saveVoiceRecordingsSetting,
screenTimeoutSetting,
screenTimeoutSettingToMs,
@@ -122,7 +121,7 @@ export type DashboardSnapshot = {
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
firmwareWarningVisible: boolean;
rawScreenshotsEnabled: boolean;
screenRecordingActive: boolean;
batteryOptimizationWarningVisible: boolean;
};
@@ -203,6 +202,7 @@ class DashboardController {
private evenAppConflictMessage = "";
private firmwareWarningMessage = "";
private batteryOptimizationWarningVisible = false;
private screenRecordingActive = false;
private displayPreview: ImageSource | null = createInitialDisplayPreview();
private silentMode = false;
// Brightness value last pushed to the glasses; see pushBrightness.
@@ -311,7 +311,7 @@ class DashboardController {
});
});
// Settings toggled from the glasses can change what the phone UI shows
// (e.g. the raw-screenshot button), so re-emit the snapshot on any change.
// (e.g. the text-setting editor), so re-emit the snapshot on any change.
onAnySettingChanged(() => {
this.emit();
// Apply a firmware-debug-flags toggle live while connected (Java dedups).
@@ -520,13 +520,45 @@ class DashboardController {
void this.communicator.setBrightness(level === null, level ?? 0).catch(() => {});
}
/** Save the current composited screen as a 4-bit grayscale PNG. */
saveScreenshot(): string {
const path = this.communicator?.saveScreenshot() ?? "";
this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: not connected");
return path;
}
/**
* Save the current composited screen as the raw headerless 4bpp frame
* buffer the wire compressor sees; test data for compression experiments.
* Begin collecting composited frames for an animated-GIF screen recording.
* Frames are captured at the same points the phone-side preview is
* refreshed, so the recording matches what the phone display showed.
*/
saveRawDashboardScreenshot(): string {
const path = this.communicator?.saveRawCompositeScreenshot() ?? "";
this.appendLog(path ? `raw screenshot saved: ${path}` : "raw screenshot skipped: not connected");
startScreenRecording(): void {
if (this.screenRecordingActive) return;
const communicator = this.communicator;
if (!communicator) {
this.appendLog("screen recording skipped: not connected");
return;
}
communicator.startScreenRecording();
// Capture the starting frame immediately rather than waiting for the
// next preview flush.
communicator.recordScreenFrame();
this.screenRecordingActive = true;
this.appendLog("screen recording started");
this.emit();
}
/** Finish the recording and save it as an animated GIF; returns the path or "". */
stopScreenRecording(): string {
if (!this.screenRecordingActive) return "";
this.screenRecordingActive = false;
let path = "";
try {
path = this.communicator?.stopScreenRecording() ?? "";
} finally {
this.emit();
}
this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames");
return path;
}
@@ -552,7 +584,7 @@ class DashboardController {
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
firmwareWarningVisible: this.firmwareWarningMessage.length > 0,
rawScreenshotsEnabled: rawScreenshotsEnabledSetting.get(),
screenRecordingActive: this.screenRecordingActive,
batteryOptimizationWarningVisible: this.batteryOptimizationWarningVisible,
};
}
@@ -908,6 +940,18 @@ class DashboardController {
this.evenHubSessionSuspended = false;
this.evenHubResumePromise = null;
// The recording's frame store lives in the communicator, so save what has
// accumulated rather than silently losing it with the connection.
if (this.screenRecordingActive) {
this.screenRecordingActive = false;
try {
const path = communicator?.stopScreenRecording() ?? "";
this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames");
} catch (error) {
this.appendLog(`screen recording save failed: ${this.formatError(error)}`);
}
}
try {
const leaseReleased = await communicator?.setFaceclawWakeLeaseEnabled(false).catch((error) => {
this.appendLog(`wake takeover lease release failed: ${this.formatError(error)}`);
@@ -1654,6 +1698,11 @@ class DashboardController {
return;
}
this.lastConnectedPreviewUpdateAtMs = now;
// The recorder captures at the same cadence the phone-side preview is
// refreshed, so the GIF matches what the phone display showed.
if (this.screenRecordingActive) {
this.communicator.recordScreenFrame();
}
const preview = this.communicator.getCompositePreview();
if (preview) {
this.setDisplayPreview(preview);
+21 -3
View File
@@ -390,10 +390,28 @@ export class FaceclawCommunicatorBridge {
return bitmap ? new ImageSource(bitmap) : null;
}
/** Save the current composite as a raw 4bpp screenshot; returns the path (empty if none). */
saveRawCompositeScreenshot(): string {
/** Save the current composite as a 4-bit grayscale PNG; returns the path (empty if none). */
saveScreenshot(): string {
if (!global.isAndroid) return "";
return String(this.communicator.saveRawCompositeScreenshot());
return String(this.communicator.saveCompositePngScreenshot());
}
/** Begin collecting composite frames for an animated-GIF screen recording. */
startScreenRecording(): void {
if (!global.isAndroid) return;
this.communicator.startScreenRecording();
}
/** Capture the current composite into the active recording; no-op when idle. */
recordScreenFrame(): void {
if (!global.isAndroid) return;
this.communicator.recordScreenFrame();
}
/** Finish the recording and save the animated GIF; returns the path (empty if no frames). */
stopScreenRecording(): string {
if (!global.isAndroid) return "";
return String(this.communicator.stopScreenRecording());
}
/**
-24
View File
@@ -1,24 +0,0 @@
import { Utils } from "@nativescript/core";
import { type GrayImage } from "../graphics/image";
declare const com: any;
/**
* Save an image as the raw headerless 576x288 4bpp frame buffer (the exact
* bytes the wire compressor sees) for compression experiments. Returns the
* absolute path of the written file; retrieve with
* adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/
*/
export function saveRawScreenshot(image: GrayImage): string {
const context = Utils.android.getApplicationContext();
if (!context) throw new Error("Android application context unavailable");
return String(
com.faceclaw.app.ScreenshotUtil.saveRawScreenshot(
context,
image.pixels.buffer,
image.width,
image.height,
),
);
}
+4 -2
View File
@@ -2,6 +2,8 @@
<ActionBar title="Faceclaw">
<ActionItem text="{{ buttonLabel }}" tap="{{ onTap }}" isEnabled="{{ canRun }}" android.position="popup" />
<ActionItem text="Configure devices" tap="{{ onConfigureTap }}" isEnabled="{{ canRun }}" android.position="popup" />
<ActionItem text="Take screenshot" tap="{{ onTakeScreenshotTap }}" android.position="popup" />
<ActionItem text="Record screen" tap="{{ onRecordScreenTap }}" android.position="popup" />
<ActionItem text="Uninstall custom firmware" tap="{{ onUninstallFirmwareTap }}" isEnabled="{{ canRun }}" android.position="popup" />
<ActionItem text="{{ showLogMenuLabel }}" tap="{{ onToggleLogTap }}" android.position="popup" />
</ActionBar>
@@ -42,7 +44,7 @@
<Button row="0" col="2" text="Microphone" tap="{{ onSyntheticMicTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<Button text="Save raw screenshot" tap="{{ onScreenshotTap }}" visibility="{{ screenshotButtonVisibility }}" />
<Button text="Stop recording" tap="{{ onStopRecordingTap }}" visibility="{{ stopRecordingButtonVisibility }}" />
</StackLayout>
<Label row="2" text="{{ status }}" class="m-t-16" textWrap="true" />
<ScrollView id="logScrollView" row="3" class="m-t-16" visibility="{{ showLogVisibility }}">
@@ -90,7 +92,7 @@
<Button row="0" col="2" text="Microphone" tap="{{ onSyntheticMicTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<Button text="Save raw screenshot" tap="{{ onScreenshotTap }}" visibility="{{ screenshotButtonVisibility }}" />
<Button text="Stop recording" tap="{{ onStopRecordingTap }}" visibility="{{ stopRecordingButtonVisibility }}" />
</StackLayout>
<Label row="1" col="0" colSpan="2" text="{{ status }}" class="m-t-16" textWrap="true" />
<ScrollView id="logScrollViewLandscape" row="2" col="0" colSpan="2" class="m-t-16" visibility="{{ showLogVisibility }}">
+30 -14
View File
@@ -17,7 +17,7 @@ export class MainViewModel extends Observable {
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
private _firmwareWarningVisible = false;
private _rawScreenshotsEnabled = false;
private _screenRecordingActive = false;
private _batteryOptimizationWarningVisible = false;
private _showLog = false;
private _phase: "disconnected" | "connecting" | "connected" | "charging" | "disconnecting" = "disconnected";
@@ -37,7 +37,7 @@ export class MainViewModel extends Observable {
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
this.firmwareWarningVisible = snapshot.firmwareWarningVisible;
this.rawScreenshotsEnabled = snapshot.rawScreenshotsEnabled;
this.screenRecordingActive = snapshot.screenRecordingActive;
this.batteryOptimizationWarningVisible = snapshot.batteryOptimizationWarningVisible;
});
}
@@ -258,27 +258,43 @@ export class MainViewModel extends Observable {
return this._firmwareWarningVisible ? "visible" : "collapse";
}
get rawScreenshotsEnabled(): boolean {
return this._rawScreenshotsEnabled;
get screenRecordingActive(): boolean {
return this._screenRecordingActive;
}
set rawScreenshotsEnabled(value: boolean) {
if (this._rawScreenshotsEnabled !== value) {
this._rawScreenshotsEnabled = value;
this.notifyPropertyChange("rawScreenshotsEnabled", value);
this.notifyPropertyChange("screenshotButtonVisibility", this.screenshotButtonVisibility);
set screenRecordingActive(value: boolean) {
if (this._screenRecordingActive !== value) {
this._screenRecordingActive = value;
this.notifyPropertyChange("screenRecordingActive", value);
this.notifyPropertyChange("stopRecordingButtonVisibility", this.stopRecordingButtonVisibility);
}
}
get screenshotButtonVisibility(): "visible" | "collapse" {
return this._rawScreenshotsEnabled ? "visible" : "collapse";
get stopRecordingButtonVisibility(): "visible" | "collapse" {
return this._screenRecordingActive ? "visible" : "collapse";
}
onScreenshotTap(): void {
onTakeScreenshotTap(): void {
try {
dashboardController.saveRawDashboardScreenshot();
dashboardController.saveScreenshot();
} catch (error) {
console.error("raw screenshot failed", error);
console.error("screenshot failed", error);
}
}
onRecordScreenTap(): void {
try {
dashboardController.startScreenRecording();
} catch (error) {
console.error("screen recording start failed", error);
}
}
onStopRecordingTap(): void {
try {
dashboardController.stopScreenRecording();
} catch (error) {
console.error("screen recording stop failed", error);
}
}
+11 -10
View File
@@ -49,8 +49,8 @@ type ConfigSettingOptions<TValue, TId extends string> = {
// Fired after any setting changes, in any isolate (storage lives in the Java
// FaceclawSettings store and broadcasts to every isolate). Lets phone-side UI
// that depends on settings toggled from the glasses (e.g. the raw-screenshot
// button) update without waiting for an unrelated snapshot emit. Delivery is
// that depends on settings toggled from the glasses (e.g. the text-setting
// editor) update without waiting for an unrelated snapshot emit. Delivery is
// asynchronous: one message-loop tick after the set().
const settingChangeListeners = new Set<() => void>();
@@ -263,14 +263,6 @@ export const voiceControlEnabledSetting = new ConfigSettingBoolean({
description: "Master switch for voice features, including wakeword detection and voice input.",
});
export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
id: "raw-screenshots-enabled",
label: "Take raw screenshots",
storageKey: "developer.rawScreenshots",
defaultValue: false,
description: "Show a button in the phone app that captures the exact frame data being sent to the glasses.",
});
export const firmwareDebugFlagsSetting = new ConfigSettingBoolean({
id: "firmware-debug-flags",
label: "Firmware debug flags",
@@ -438,6 +430,15 @@ export const terminalLaunchPresetsSetting = new ConfigSettingString({
"Comma-separated names of g2mirror launch presets that can be started from the glasses. Presets are defined in the server's config; the wire protocol has no way to list them, so name them here. The default server config defines \"shell\".",
});
export const terminalWakeOnBellSetting = new ConfigSettingBoolean({
id: "terminal-wake-on-bell",
label: "Wake glasses on terminal bell",
storageKey: "terminal.wakeOnBell",
defaultValue: false,
description:
"When a terminal rings its bell while the glasses are asleep, wake them and focus that terminal's window (or the terminals list if it has no window open).",
});
export const nightscoutSiteUrlSetting = new ConfigSettingString({
id: "nightscout-site-url",
label: "Nightscout site URL",
+2 -2
View File
@@ -13,13 +13,13 @@ import {
openAiApiKeySetting,
enumSettingMenuItem,
firmwareDebugFlagsSetting,
rawScreenshotsEnabledSetting,
saveVoiceRecordingsSetting,
suspendEvenHubWhenScreenOffSetting,
terminalAuthTokenSetting,
terminalHostSetting,
terminalLaunchPresetsSetting,
terminalPortSetting,
terminalWakeOnBellSetting,
textSettingMenuItem,
timeFormatSetting,
toggleSettingMenuItem,
@@ -83,12 +83,12 @@ function settingsSections(): SettingsSection[] {
textSettingMenuItem(terminalPortSetting),
textSettingMenuItem(terminalAuthTokenSetting),
textSettingMenuItem(terminalLaunchPresetsSetting),
toggleSettingMenuItem(terminalWakeOnBellSetting),
],
},
{
label: "Developer",
items: [
toggleSettingMenuItem(rawScreenshotsEnabledSetting),
toggleSettingMenuItem(saveVoiceRecordingsSetting),
toggleSettingMenuItem(firmwareDebugFlagsSetting),
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
+17
View File
@@ -28,6 +28,15 @@ export type WorkerAppReply =
type: "focus-window";
windowId: string;
}
| {
/**
* Wake the glasses and focus one of the app's windows (e.g. a terminal
* bell with wake-on-bell enabled). Ignored unless the screen is off, so
* it can never steal focus from an in-use screen.
*/
type: "wake-window";
windowId: string;
}
| {
/** App-initiated window (e.g. a terminal view opened from the hub list). */
type: "open-window-request";
@@ -149,6 +158,14 @@ export class WorkerAppHost {
this.options.requestShellRender();
}
break;
case "wake-window":
if (this.openWindows.has(message.windowId) && !shell.isScreenOn()) {
// Focus first so wake's foreground-window refresh hits the target.
shell.focusWindow(message.windowId);
shell.wake("window");
this.options.requestShellRender();
}
break;
case "open-window-request":
this.openWindow({
windowId: message.windowId,
+88 -10
View File
@@ -30,6 +30,7 @@ import {
terminalHostSetting,
terminalLaunchPresetsSetting,
terminalPortSetting,
terminalWakeOnBellSetting,
} from "../ui/dashboard-settings";
import { TerminalEmulator } from "../ui/apps/terminal-emulator";
import type { DashboardInputEvent } from "../ui/layers";
@@ -76,6 +77,13 @@ type HubWindow = BaseWindow & {
kind: "hub";
selectedIndex: number;
scrollRow: number;
/**
* Session sockets in display order, captured when the window last became
* visible so the list doesn't reshuffle under the user while it's open.
* Cleared on foreground/screen-on; orderedSessions() rebuilds it lazily,
* sorting by recency and appending sessions that appear while visible.
*/
sessionOrder: string[];
};
type ViewWindow = BaseWindow & {
@@ -179,6 +187,13 @@ let controlClient: G2MirrorClient | null = null;
let controlState: G2MirrorState | null = null;
let controlUnsubscribers: Array<() => void> = [];
// When each session (by socket) last "updated", unix epoch ms: bells, title
// changes, and terminal output from open views. Sessions first seen in the
// initial list after (re)connect are seeded from their last bell, since we
// weren't watching; ones appearing later were just created, so they get "now".
const sessionRecency = new Map<string, number>();
let controlSessionsSeeded = false;
function post(message: WorkerAppReply): void {
global.postMessage(message);
}
@@ -225,6 +240,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
window.foreground = message.foreground;
window.focused = message.focused;
if (window.foreground && window.kind === "view") activeViewId = window.windowId;
if (window.foreground && window.kind === "hub") window.sessionOrder = [];
if (window.foreground) renderAndSubmit(window, 0);
break;
}
@@ -232,7 +248,10 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
screenOn = message.on;
if (screenOn) {
for (const window of windows.values()) {
if (window.foreground) renderAndSubmit(window, 0);
if (!window.foreground) continue;
// Waking counts as becoming visible: let the session list re-sort.
if (window.kind === "hub") window.sessionOrder = [];
renderAndSubmit(window, 0);
}
}
break;
@@ -256,8 +275,8 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
// in the dashboard's Settings menu, which lives in the main isolate).
onSettingsStoreChanged((key) => {
if (!key.startsWith("terminal.")) return;
// The preset list only feeds menus and tool replies; no reconnect needed.
if (key === "terminal.launchPresets") return;
// Settings that don't affect the connection; no reconnect needed.
if (key === "terminal.launchPresets" || key === "terminal.wakeOnBell") return;
if (gridCols > 0) {
startControlClient();
}
@@ -287,6 +306,7 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb
menu: null,
selectedIndex: 0,
scrollRow: 0,
sessionOrder: [],
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
@@ -333,18 +353,42 @@ function startControlClient(): void {
const client = new G2MirrorClient(options);
controlClient = client;
controlState = client.state();
controlSessionsSeeded = false;
controlUnsubscribers.push(
client.onStateChange((state) => {
noteSessionListRecency(state);
controlState = state;
renderHubWindows();
}),
client.onBell((socket) => {
routeBell(socket);
client.onBell((socket, lastBellAtMs) => {
routeBell(socket, lastBellAtMs);
}),
client.onTitle((socket) => {
noteSessionUpdated(socket);
}),
);
client.start();
}
function noteSessionUpdated(socket: string): void {
sessionRecency.set(socket, Date.now());
}
function noteSessionListRecency(state: G2MirrorState): void {
for (const session of state.sessions) {
const known = sessionRecency.get(session.socket);
if (known === undefined) {
sessionRecency.set(
session.socket,
Math.max(session.lastBellAt ?? 0, controlSessionsSeeded ? Date.now() : 0),
);
} else if ((session.lastBellAt ?? 0) > known) {
sessionRecency.set(session.socket, session.lastBellAt!);
}
}
if (state.sessions.length) controlSessionsSeeded = true;
}
function stopControlClient(): void {
for (const unsubscribe of controlUnsubscribers.splice(0)) {
unsubscribe();
@@ -354,12 +398,22 @@ function stopControlClient(): void {
controlState = null;
}
function routeBell(socket: string): void {
function routeBell(socket: string, lastBellAtMs: number): void {
const known = sessionRecency.get(socket) ?? 0;
if (lastBellAtMs > known) sessionRecency.set(socket, lastBellAtMs);
for (const window of windows.values()) {
if (window.kind === "view" && window.socket === socket && !window.foreground) {
post({ type: "set-attention", windowId: window.windowId, attention: true });
}
}
if (terminalWakeOnBellSetting.get()) {
// Wake to the belling session's view window, or the hub if it has none.
// The shell drops the message unless the glasses are actually asleep.
const viewId = viewWindowIdForSocket(socket);
const hubId = [...windows.values()].find((window) => window.kind === "hub")?.windowId ?? null;
const target = viewId ?? hubId;
if (target) post({ type: "wake-window", windowId: target });
}
}
function renderHubWindows(): void {
@@ -434,6 +488,7 @@ function createViewWindow(
window.emulator.reset();
}
window.receivedData = true;
noteSessionUpdated(window.socket);
window.emulator.write(data, () => scheduleRender(window));
}),
client.onSessionDetached((reason) => {
@@ -594,13 +649,36 @@ type HubItem = {
onSelect?: () => void;
};
function hubItems(): HubItem[] {
/**
* The control connection's sessions in this hub window's display order:
* most-recently-updated first as of when the window became visible, with
* sessions that appeared since then at the end. The captured order lives in
* window.sessionOrder (cleared when the window becomes visible) so the list
* doesn't reshuffle while the user is looking at it.
*/
function orderedSessions(window: HubWindow): G2MirrorSession[] {
const sessions = controlState?.sessions ?? [];
const position = new Map<string, number>();
for (const socket of window.sessionOrder) {
if (!position.has(socket)) position.set(socket, position.size);
}
const fresh = sessions
.filter((session) => !position.has(session.socket))
.sort((a, b) => (sessionRecency.get(b.socket) ?? 0) - (sessionRecency.get(a.socket) ?? 0));
for (const session of fresh) {
position.set(session.socket, position.size);
window.sessionOrder.push(session.socket);
}
return sessions.slice().sort((a, b) => position.get(a.socket)! - position.get(b.socket)!);
}
function hubItems(window: HubWindow): HubItem[] {
const items: HubItem[] = [];
const state = controlState;
const phase = state?.phase ?? "idle";
if (phase === "connected" || phase === "attached") {
for (const session of state?.sessions ?? []) {
for (const session of orderedSessions(window)) {
const openWindowId = viewWindowIdForSocket(session.socket);
items.push({
label: openWindowId ? `${sessionLabel(session)} [open]` : sessionLabel(session),
@@ -631,7 +709,7 @@ function hubItems(): HubItem[] {
}
function handleHubInput(window: HubWindow, event: DashboardInputEvent, frameId: number): void {
const items = hubItems();
const items = hubItems(window);
switch (event.type) {
case "scroll-up":
window.selectedIndex = Math.max(0, window.selectedIndex - 1);
@@ -746,7 +824,7 @@ function paintHub(window: HubWindow): GrayImage {
listTop += 28;
}
const items = hubItems();
const items = hubItems(window);
window.selectedIndex = Math.max(0, Math.min(window.selectedIndex, items.length - 1));
const visibleRowCount = Math.max(1, ((viewportHeight - 30 - listTop) / HUB_ROW_HEIGHT) | 0);
window.scrollRow = scrollToKeepSelectionVisible(window.scrollRow, window.selectedIndex, visibleRowCount, items.length);