diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
index f919e0eb..45810fa7 100644
--- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
@@ -437,14 +437,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
}
- /** Save the current composite as a raw 4bpp screenshot; returns the path or "". */
- public String saveRawCompositeScreenshot() throws java.io.IOException {
+ /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
+ public String saveCompositePngScreenshot() throws java.io.IOException {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return "";
}
- return ScreenshotUtil.saveRawScreenshot(
- appContext, java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height);
+ return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height);
+ }
+
+ // Active animated-GIF screen recording, or null when idle. Frames are
+ // pushed by recordScreenFrame(), which the TS side calls at each
+ // phone-preview flush.
+ private volatile GifScreenRecorder screenRecorder;
+
+ /** Begin collecting composite frames for an animated-GIF screen recording. */
+ public void startScreenRecording() {
+ screenRecorder = new GifScreenRecorder();
+ }
+
+ /** Capture the current composite into the active recording; no-op when idle. */
+ public void recordScreenFrame() {
+ GifScreenRecorder recorder = screenRecorder;
+ if (recorder == null) {
+ return;
+ }
+ SurfaceCompositor.Composite composite = compositor.previewComposite();
+ if (composite == null) {
+ return;
+ }
+ recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis());
+ }
+
+ /** Finish the recording and save it as an animated GIF; returns the path or "". */
+ public String stopScreenRecording() throws java.io.IOException {
+ GifScreenRecorder recorder = screenRecorder;
+ screenRecorder = null;
+ if (recorder == null) {
+ return "";
+ }
+ if (recorder.isOverflowed()) {
+ logLine("screen recording hit its frame cap; the tail was dropped");
+ }
+ return recorder.save(appContext);
}
/**
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java
new file mode 100644
index 00000000..cfba7631
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/GifScreenRecorder.java
@@ -0,0 +1,273 @@
+package com.faceclaw.app;
+
+import android.content.Context;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.zip.DataFormatException;
+import java.util.zip.Inflater;
+
+/**
+ * Accumulates composited screen frames and saves them as a looping animated
+ * GIF with a 16-entry grayscale palette. Stored frames are deflate-compressed
+ * (a typical UI frame shrinks from 166KB of indices to a few KB) so long
+ * recordings stay affordable in memory; consecutive identical frames are
+ * deduplicated, with the elapsed time folded into the next frame's delay.
+ */
+public final class GifScreenRecorder {
+ /** Backstop against unbounded growth if a recording is left running. */
+ private static final int MAX_FRAMES = 3600;
+ /** Displayed duration of the final frame, which has no successor. */
+ private static final int LAST_FRAME_DELAY_CS = 100;
+ /** Below this, GIF renderers historically clamp or ignore the delay. */
+ private static final int MIN_FRAME_DELAY_CS = 2;
+
+ private static final class Frame {
+ final byte[] deflatedIndices;
+ final long timestampMs;
+
+ Frame(byte[] deflatedIndices, long timestampMs) {
+ this.deflatedIndices = deflatedIndices;
+ this.timestampMs = timestampMs;
+ }
+ }
+
+ private final List frames = new ArrayList<>();
+ /** Undeflated copy of the newest frame, kept for duplicate detection. */
+ private byte[] lastIndices;
+ private int width;
+ private int height;
+ private boolean overflowed;
+
+ /**
+ * Append the current screen. gray is 8bpp; only the top nibble (the
+ * display's native depth, and the palette index) is kept.
+ */
+ public synchronized void addFrame(byte[] gray, int width, int height, long timestampMs) {
+ if (gray == null || width <= 0 || height <= 0 || gray.length < width * height) {
+ return;
+ }
+ if (frames.isEmpty()) {
+ this.width = width;
+ this.height = height;
+ } else if (width != this.width || height != this.height) {
+ return;
+ }
+ byte[] indices = new byte[width * height];
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = (byte) ((gray[i] & 0xff) >>> 4);
+ }
+ if (lastIndices != null && Arrays.equals(indices, lastIndices)) {
+ return;
+ }
+ if (frames.size() >= MAX_FRAMES) {
+ overflowed = true;
+ return;
+ }
+ frames.add(new Frame(ScreenshotUtil.zlibDeflate(indices), timestampMs));
+ lastIndices = indices;
+ }
+
+ /** True if frames were dropped because the recording hit MAX_FRAMES. */
+ public synchronized boolean isOverflowed() {
+ return overflowed;
+ }
+
+ /**
+ * Encode the recording as an animated GIF in the screenshots directory.
+ * Returns the absolute path, or "" if no frames were captured.
+ */
+ public synchronized String save(Context context) throws IOException {
+ if (frames.isEmpty()) {
+ return "";
+ }
+ File file = new File(ScreenshotUtil.ensureScreenshotsDir(context), "recording-" + ScreenshotUtil.timestamp() + ".gif");
+ try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
+ writeGif(out);
+ }
+ return file.getAbsolutePath();
+ }
+
+ private void writeGif(OutputStream out) throws IOException {
+ out.write('G');
+ out.write('I');
+ out.write('F');
+ out.write('8');
+ out.write('9');
+ out.write('a');
+ // Logical screen descriptor: global color table present, 8 bits of
+ // color resolution, 2^(3+1)=16 palette entries.
+ writeShortLE(out, width);
+ writeShortLE(out, height);
+ out.write(0x80 | 0x70 | 0x03);
+ out.write(0); // background color index
+ out.write(0); // pixel aspect ratio
+ for (int i = 0; i < 16; i++) {
+ int v = i * 17;
+ out.write(v);
+ out.write(v);
+ out.write(v);
+ }
+ // Netscape application extension: loop forever.
+ out.write(0x21);
+ out.write(0xff);
+ out.write(11);
+ out.write("NETSCAPE2.0".getBytes("US-ASCII"));
+ out.write(3);
+ out.write(1);
+ writeShortLE(out, 0);
+ out.write(0);
+
+ for (int i = 0; i < frames.size(); i++) {
+ Frame frame = frames.get(i);
+ // Each frame displays until the next one was captured; the last
+ // frame has no successor, so give it a nominal one-second hold.
+ int delayCs = LAST_FRAME_DELAY_CS;
+ if (i + 1 < frames.size()) {
+ long deltaMs = frames.get(i + 1).timestampMs - frame.timestampMs;
+ delayCs = (int) Math.max(MIN_FRAME_DELAY_CS, Math.min(0xffff, Math.round(deltaMs / 10.0)));
+ }
+ // Graphic control extension: no disposal, no transparency.
+ out.write(0x21);
+ out.write(0xf9);
+ out.write(4);
+ out.write(0x04); // disposal method 1 (do not dispose)
+ writeShortLE(out, delayCs);
+ out.write(0); // transparent color index (unused)
+ out.write(0);
+ // Image descriptor: full frame, no local color table.
+ out.write(0x2c);
+ writeShortLE(out, 0);
+ writeShortLE(out, 0);
+ writeShortLE(out, width);
+ writeShortLE(out, height);
+ out.write(0);
+ writeLzwImageData(out, inflate(frame.deflatedIndices, width * height));
+ }
+ out.write(0x3b); // trailer
+ }
+
+ private static byte[] inflate(byte[] deflated, int size) throws IOException {
+ Inflater inflater = new Inflater();
+ inflater.setInput(deflated);
+ byte[] data = new byte[size];
+ try {
+ int offset = 0;
+ while (offset < size && !inflater.finished()) {
+ int n = inflater.inflate(data, offset, size - offset);
+ if (n == 0) break;
+ offset += n;
+ }
+ if (offset != size) {
+ throw new IOException("recorded frame truncated: " + offset + " of " + size);
+ }
+ } catch (DataFormatException e) {
+ throw new IOException("recorded frame corrupt", e);
+ } finally {
+ inflater.end();
+ }
+ return data;
+ }
+
+ /** GIF LZW compression of a 4-bit index stream (minimum code size 4). */
+ private static void writeLzwImageData(OutputStream out, byte[] indices) throws IOException {
+ final int minCodeSize = 4;
+ out.write(minCodeSize);
+ SubBlockBitWriter writer = new SubBlockBitWriter(out);
+ final int clearCode = 1 << minCodeSize;
+ final int eoiCode = clearCode + 1;
+ int codeSize = minCodeSize + 1;
+ int nextCode = eoiCode + 1;
+ HashMap table = new HashMap<>();
+ writer.writeCode(clearCode, codeSize);
+ int prefix = indices[0] & 0xff;
+ for (int i = 1; i < indices.length; i++) {
+ int k = indices[i] & 0xff;
+ int key = (prefix << 8) | k;
+ Integer code = table.get(key);
+ if (code != null) {
+ prefix = code;
+ continue;
+ }
+ writer.writeCode(prefix, codeSize);
+ if (nextCode == 0x1000) {
+ writer.writeCode(clearCode, codeSize);
+ table.clear();
+ codeSize = minCodeSize + 1;
+ nextCode = eoiCode + 1;
+ } else {
+ if (nextCode >= (1 << codeSize)) {
+ codeSize++;
+ }
+ table.put(key, nextCode++);
+ }
+ prefix = k;
+ }
+ writer.writeCode(prefix, codeSize);
+ writer.writeCode(eoiCode, codeSize);
+ writer.finish();
+ }
+
+ /**
+ * Packs LZW codes LSB-first into bytes and the bytes into the 255-byte
+ * data sub-blocks GIF image data is carried in.
+ */
+ private static final class SubBlockBitWriter {
+ private final OutputStream out;
+ private final byte[] block = new byte[255];
+ private int blockLength;
+ private int bitBuffer;
+ private int bitCount;
+
+ SubBlockBitWriter(OutputStream out) {
+ this.out = out;
+ }
+
+ void writeCode(int code, int codeSize) throws IOException {
+ bitBuffer |= code << bitCount;
+ bitCount += codeSize;
+ while (bitCount >= 8) {
+ writeByte(bitBuffer & 0xff);
+ bitBuffer >>>= 8;
+ bitCount -= 8;
+ }
+ }
+
+ void finish() throws IOException {
+ if (bitCount > 0) {
+ writeByte(bitBuffer & 0xff);
+ bitBuffer = 0;
+ bitCount = 0;
+ }
+ flushBlock();
+ out.write(0); // block terminator
+ }
+
+ private void writeByte(int b) throws IOException {
+ block[blockLength++] = (byte) b;
+ if (blockLength == 255) {
+ flushBlock();
+ }
+ }
+
+ private void flushBlock() throws IOException {
+ if (blockLength == 0) return;
+ out.write(blockLength);
+ out.write(block, 0, blockLength);
+ blockLength = 0;
+ }
+ }
+
+ private static void writeShortLE(OutputStream out, int value) throws IOException {
+ out.write(value & 0xff);
+ out.write((value >>> 8) & 0xff);
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java
index f80cf7ef..9ffdcd1b 100644
--- a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java
@@ -2,43 +2,123 @@ package com.faceclaw.app;
import android.content.Context;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.nio.ByteBuffer;
+import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
+import java.util.zip.CRC32;
+import java.util.zip.Deflater;
/**
- * Saves raw screenshots for compression experiments. The output is the exact
- * headerless top-down 4bpp frame buffer that gets zlib-compressed on the wire
- * (build4bppBmp followed by pack4bppFromBmp), deliberately untransformed — no
- * resize, no gamma, no header.
+ * Saves screenshots of the composited screen as 4-bit grayscale PNGs (the
+ * display's native depth). Retrieve with
+ * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/
*/
public final class ScreenshotUtil {
private ScreenshotUtil() {}
/** Returns the absolute path of the written file. */
- public static String saveRawScreenshot(Context context, ByteBuffer image8bpp, int width, int height) throws java.io.IOException {
- if (image8bpp == null || width <= 0 || height <= 0 || image8bpp.remaining() < width * height) {
+ public static String savePngScreenshot(Context context, byte[] gray, int width, int height) throws IOException {
+ if (gray == null || width <= 0 || height <= 0 || gray.length < width * height) {
throw new IllegalArgumentException("invalid screenshot buffer");
}
- byte[] gray = new byte[image8bpp.remaining()];
- image8bpp.get(gray);
- byte[] packed = BmpUtil.pack4bppFromBmp(BmpUtil.build4bppBmp(gray, width, height));
- if (packed.length == 0) {
- throw new IllegalStateException("failed to pack 4bpp frame");
- }
-
- File dir = new File(context.getExternalFilesDir(null), "screenshots");
- if (!dir.exists() && !dir.mkdirs()) {
- throw new java.io.IOException("failed to create " + dir);
- }
- String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date());
- File file = new File(dir, "screen-" + stamp + "-" + width + "x" + height + ".4bpp");
+ byte[] png = encode4BitGrayPng(gray, width, height);
+ File file = new File(ensureScreenshotsDir(context), "screen-" + timestamp() + ".png");
try (FileOutputStream out = new FileOutputStream(file)) {
- out.write(packed);
+ out.write(png);
}
return file.getAbsolutePath();
}
+
+ static File ensureScreenshotsDir(Context context) throws IOException {
+ File dir = new File(context.getExternalFilesDir(null), "screenshots");
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException("failed to create " + dir);
+ }
+ return dir;
+ }
+
+ static String timestamp() {
+ return new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date());
+ }
+
+ /**
+ * Encode an 8bpp grayscale buffer as a bit-depth-4 grayscale PNG, keeping
+ * the top nibble of each pixel (the same quantization the wire format uses).
+ */
+ static byte[] encode4BitGrayPng(byte[] gray, int width, int height) {
+ // Raw scanlines: one filter byte (0 = None) then two pixels per byte,
+ // leftmost pixel in the high nibble.
+ int rowBytes = (width + 1) / 2;
+ byte[] raw = new byte[height * (1 + rowBytes)];
+ int p = 0;
+ for (int y = 0; y < height; y++) {
+ raw[p++] = 0;
+ int row = y * width;
+ for (int x = 0; x < width; x += 2) {
+ int hi = (gray[row + x] & 0xff) >>> 4;
+ int lo = (x + 1 < width) ? (gray[row + x + 1] & 0xff) >>> 4 : 0;
+ raw[p++] = (byte) ((hi << 4) | lo);
+ }
+ }
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ out.write(0x89);
+ out.write('P');
+ out.write('N');
+ out.write('G');
+ out.write(0x0d);
+ out.write(0x0a);
+ out.write(0x1a);
+ out.write(0x0a);
+
+ ByteArrayOutputStream ihdr = new ByteArrayOutputStream();
+ writeIntBE(ihdr, width);
+ writeIntBE(ihdr, height);
+ ihdr.write(4); // bit depth
+ ihdr.write(0); // color type: grayscale
+ ihdr.write(0); // compression
+ ihdr.write(0); // filter
+ ihdr.write(0); // interlace
+ writeChunk(out, "IHDR", ihdr.toByteArray());
+ writeChunk(out, "IDAT", zlibDeflate(raw));
+ writeChunk(out, "IEND", new byte[0]);
+ return out.toByteArray();
+ }
+
+ static byte[] zlibDeflate(byte[] data) {
+ Deflater deflater = new Deflater();
+ deflater.setInput(data);
+ deflater.finish();
+ ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, data.length / 8));
+ byte[] buffer = new byte[8192];
+ while (!deflater.finished()) {
+ out.write(buffer, 0, deflater.deflate(buffer));
+ }
+ deflater.end();
+ return out.toByteArray();
+ }
+
+ private static void writeChunk(ByteArrayOutputStream out, String type, byte[] data) {
+ writeIntBE(out, data.length);
+ CRC32 crc = new CRC32();
+ for (int i = 0; i < 4; i++) {
+ int b = type.charAt(i);
+ out.write(b);
+ crc.update(b);
+ }
+ out.write(data, 0, data.length);
+ crc.update(data);
+ writeIntBE(out, (int) crc.getValue());
+ }
+
+ private static void writeIntBE(ByteArrayOutputStream out, int value) {
+ out.write((value >>> 24) & 0xff);
+ out.write((value >>> 16) & 0xff);
+ out.write((value >>> 8) & 0xff);
+ out.write(value & 0xff);
+ }
}
diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts
index a38e9dbe..51777e46 100644
--- a/app/g2/dashboard-controller.ts
+++ b/app/g2/dashboard-controller.ts
@@ -90,7 +90,6 @@ import {
firmwareDebugFlagsSetting,
nightscoutSiteUrlSetting,
onAnySettingChanged,
- rawScreenshotsEnabledSetting,
saveVoiceRecordingsSetting,
screenTimeoutSetting,
screenTimeoutSettingToMs,
@@ -122,7 +121,7 @@ export type DashboardSnapshot = {
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
firmwareWarningVisible: boolean;
- rawScreenshotsEnabled: boolean;
+ screenRecordingActive: boolean;
batteryOptimizationWarningVisible: boolean;
};
@@ -203,6 +202,7 @@ class DashboardController {
private evenAppConflictMessage = "";
private firmwareWarningMessage = "";
private batteryOptimizationWarningVisible = false;
+ private screenRecordingActive = false;
private displayPreview: ImageSource | null = createInitialDisplayPreview();
private silentMode = false;
// Brightness value last pushed to the glasses; see pushBrightness.
@@ -311,7 +311,7 @@ class DashboardController {
});
});
// Settings toggled from the glasses can change what the phone UI shows
- // (e.g. the raw-screenshot button), so re-emit the snapshot on any change.
+ // (e.g. the text-setting editor), so re-emit the snapshot on any change.
onAnySettingChanged(() => {
this.emit();
// Apply a firmware-debug-flags toggle live while connected (Java dedups).
@@ -520,13 +520,45 @@ class DashboardController {
void this.communicator.setBrightness(level === null, level ?? 0).catch(() => {});
}
+ /** Save the current composited screen as a 4-bit grayscale PNG. */
+ saveScreenshot(): string {
+ const path = this.communicator?.saveScreenshot() ?? "";
+ this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: not connected");
+ return path;
+ }
+
/**
- * Save the current composited screen as the raw headerless 4bpp frame
- * buffer the wire compressor sees; test data for compression experiments.
+ * Begin collecting composited frames for an animated-GIF screen recording.
+ * Frames are captured at the same points the phone-side preview is
+ * refreshed, so the recording matches what the phone display showed.
*/
- saveRawDashboardScreenshot(): string {
- const path = this.communicator?.saveRawCompositeScreenshot() ?? "";
- this.appendLog(path ? `raw screenshot saved: ${path}` : "raw screenshot skipped: not connected");
+ startScreenRecording(): void {
+ if (this.screenRecordingActive) return;
+ const communicator = this.communicator;
+ if (!communicator) {
+ this.appendLog("screen recording skipped: not connected");
+ return;
+ }
+ communicator.startScreenRecording();
+ // Capture the starting frame immediately rather than waiting for the
+ // next preview flush.
+ communicator.recordScreenFrame();
+ this.screenRecordingActive = true;
+ this.appendLog("screen recording started");
+ this.emit();
+ }
+
+ /** Finish the recording and save it as an animated GIF; returns the path or "". */
+ stopScreenRecording(): string {
+ if (!this.screenRecordingActive) return "";
+ this.screenRecordingActive = false;
+ let path = "";
+ try {
+ path = this.communicator?.stopScreenRecording() ?? "";
+ } finally {
+ this.emit();
+ }
+ this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames");
return path;
}
@@ -552,7 +584,7 @@ class DashboardController {
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
firmwareWarningVisible: this.firmwareWarningMessage.length > 0,
- rawScreenshotsEnabled: rawScreenshotsEnabledSetting.get(),
+ screenRecordingActive: this.screenRecordingActive,
batteryOptimizationWarningVisible: this.batteryOptimizationWarningVisible,
};
}
@@ -908,6 +940,18 @@ class DashboardController {
this.evenHubSessionSuspended = false;
this.evenHubResumePromise = null;
+ // The recording's frame store lives in the communicator, so save what has
+ // accumulated rather than silently losing it with the connection.
+ if (this.screenRecordingActive) {
+ this.screenRecordingActive = false;
+ try {
+ const path = communicator?.stopScreenRecording() ?? "";
+ this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames");
+ } catch (error) {
+ this.appendLog(`screen recording save failed: ${this.formatError(error)}`);
+ }
+ }
+
try {
const leaseReleased = await communicator?.setFaceclawWakeLeaseEnabled(false).catch((error) => {
this.appendLog(`wake takeover lease release failed: ${this.formatError(error)}`);
@@ -1654,6 +1698,11 @@ class DashboardController {
return;
}
this.lastConnectedPreviewUpdateAtMs = now;
+ // The recorder captures at the same cadence the phone-side preview is
+ // refreshed, so the GIF matches what the phone display showed.
+ if (this.screenRecordingActive) {
+ this.communicator.recordScreenFrame();
+ }
const preview = this.communicator.getCompositePreview();
if (preview) {
this.setDisplayPreview(preview);
diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts
index f1f196ff..ce896c89 100644
--- a/app/native/faceclaw-communicator.ts
+++ b/app/native/faceclaw-communicator.ts
@@ -390,10 +390,28 @@ export class FaceclawCommunicatorBridge {
return bitmap ? new ImageSource(bitmap) : null;
}
- /** Save the current composite as a raw 4bpp screenshot; returns the path (empty if none). */
- saveRawCompositeScreenshot(): string {
+ /** Save the current composite as a 4-bit grayscale PNG; returns the path (empty if none). */
+ saveScreenshot(): string {
if (!global.isAndroid) return "";
- return String(this.communicator.saveRawCompositeScreenshot());
+ return String(this.communicator.saveCompositePngScreenshot());
+ }
+
+ /** Begin collecting composite frames for an animated-GIF screen recording. */
+ startScreenRecording(): void {
+ if (!global.isAndroid) return;
+ this.communicator.startScreenRecording();
+ }
+
+ /** Capture the current composite into the active recording; no-op when idle. */
+ recordScreenFrame(): void {
+ if (!global.isAndroid) return;
+ this.communicator.recordScreenFrame();
+ }
+
+ /** Finish the recording and save the animated GIF; returns the path (empty if no frames). */
+ stopScreenRecording(): string {
+ if (!global.isAndroid) return "";
+ return String(this.communicator.stopScreenRecording());
}
/**
diff --git a/app/native/raw-screenshot.ts b/app/native/raw-screenshot.ts
deleted file mode 100644
index 39e0ebc4..00000000
--- a/app/native/raw-screenshot.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import { Utils } from "@nativescript/core";
-
-import { type GrayImage } from "../graphics/image";
-
-declare const com: any;
-
-/**
- * Save an image as the raw headerless 576x288 4bpp frame buffer (the exact
- * bytes the wire compressor sees) for compression experiments. Returns the
- * absolute path of the written file; retrieve with
- * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/
- */
-export function saveRawScreenshot(image: GrayImage): string {
- const context = Utils.android.getApplicationContext();
- if (!context) throw new Error("Android application context unavailable");
- return String(
- com.faceclaw.app.ScreenshotUtil.saveRawScreenshot(
- context,
- image.pixels.buffer,
- image.width,
- image.height,
- ),
- );
-}
diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml
index 6d34beb6..bf7bc295 100644
--- a/app/phone-ui/main-page.xml
+++ b/app/phone-ui/main-page.xml
@@ -2,6 +2,8 @@
+
+
@@ -42,7 +44,7 @@
-
+
@@ -90,7 +92,7 @@
-
+
diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts
index d21638cd..ffbb6760 100644
--- a/app/phone-ui/main-view-model.ts
+++ b/app/phone-ui/main-view-model.ts
@@ -17,7 +17,7 @@ export class MainViewModel extends Observable {
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
private _firmwareWarningVisible = false;
- private _rawScreenshotsEnabled = false;
+ private _screenRecordingActive = false;
private _batteryOptimizationWarningVisible = false;
private _showLog = false;
private _phase: "disconnected" | "connecting" | "connected" | "charging" | "disconnecting" = "disconnected";
@@ -37,7 +37,7 @@ export class MainViewModel extends Observable {
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
this.firmwareWarningVisible = snapshot.firmwareWarningVisible;
- this.rawScreenshotsEnabled = snapshot.rawScreenshotsEnabled;
+ this.screenRecordingActive = snapshot.screenRecordingActive;
this.batteryOptimizationWarningVisible = snapshot.batteryOptimizationWarningVisible;
});
}
@@ -258,27 +258,43 @@ export class MainViewModel extends Observable {
return this._firmwareWarningVisible ? "visible" : "collapse";
}
- get rawScreenshotsEnabled(): boolean {
- return this._rawScreenshotsEnabled;
+ get screenRecordingActive(): boolean {
+ return this._screenRecordingActive;
}
- set rawScreenshotsEnabled(value: boolean) {
- if (this._rawScreenshotsEnabled !== value) {
- this._rawScreenshotsEnabled = value;
- this.notifyPropertyChange("rawScreenshotsEnabled", value);
- this.notifyPropertyChange("screenshotButtonVisibility", this.screenshotButtonVisibility);
+ set screenRecordingActive(value: boolean) {
+ if (this._screenRecordingActive !== value) {
+ this._screenRecordingActive = value;
+ this.notifyPropertyChange("screenRecordingActive", value);
+ this.notifyPropertyChange("stopRecordingButtonVisibility", this.stopRecordingButtonVisibility);
}
}
- get screenshotButtonVisibility(): "visible" | "collapse" {
- return this._rawScreenshotsEnabled ? "visible" : "collapse";
+ get stopRecordingButtonVisibility(): "visible" | "collapse" {
+ return this._screenRecordingActive ? "visible" : "collapse";
}
- onScreenshotTap(): void {
+ onTakeScreenshotTap(): void {
try {
- dashboardController.saveRawDashboardScreenshot();
+ dashboardController.saveScreenshot();
} catch (error) {
- console.error("raw screenshot failed", error);
+ console.error("screenshot failed", error);
+ }
+ }
+
+ onRecordScreenTap(): void {
+ try {
+ dashboardController.startScreenRecording();
+ } catch (error) {
+ console.error("screen recording start failed", error);
+ }
+ }
+
+ onStopRecordingTap(): void {
+ try {
+ dashboardController.stopScreenRecording();
+ } catch (error) {
+ console.error("screen recording stop failed", error);
}
}
diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts
index ab3329bf..eae077ea 100644
--- a/app/ui/dashboard-settings.ts
+++ b/app/ui/dashboard-settings.ts
@@ -49,8 +49,8 @@ type ConfigSettingOptions = {
// Fired after any setting changes, in any isolate (storage lives in the Java
// FaceclawSettings store and broadcasts to every isolate). Lets phone-side UI
-// that depends on settings toggled from the glasses (e.g. the raw-screenshot
-// button) update without waiting for an unrelated snapshot emit. Delivery is
+// that depends on settings toggled from the glasses (e.g. the text-setting
+// editor) update without waiting for an unrelated snapshot emit. Delivery is
// asynchronous: one message-loop tick after the set().
const settingChangeListeners = new Set<() => void>();
@@ -263,14 +263,6 @@ export const voiceControlEnabledSetting = new ConfigSettingBoolean({
description: "Master switch for voice features, including wakeword detection and voice input.",
});
-export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
- id: "raw-screenshots-enabled",
- label: "Take raw screenshots",
- storageKey: "developer.rawScreenshots",
- defaultValue: false,
- description: "Show a button in the phone app that captures the exact frame data being sent to the glasses.",
-});
-
export const firmwareDebugFlagsSetting = new ConfigSettingBoolean({
id: "firmware-debug-flags",
label: "Firmware debug flags",
@@ -438,6 +430,15 @@ export const terminalLaunchPresetsSetting = new ConfigSettingString({
"Comma-separated names of g2mirror launch presets that can be started from the glasses. Presets are defined in the server's config; the wire protocol has no way to list them, so name them here. The default server config defines \"shell\".",
});
+export const terminalWakeOnBellSetting = new ConfigSettingBoolean({
+ id: "terminal-wake-on-bell",
+ label: "Wake glasses on terminal bell",
+ storageKey: "terminal.wakeOnBell",
+ defaultValue: false,
+ description:
+ "When a terminal rings its bell while the glasses are asleep, wake them and focus that terminal's window (or the terminals list if it has no window open).",
+});
+
export const nightscoutSiteUrlSetting = new ConfigSettingString({
id: "nightscout-site-url",
label: "Nightscout site URL",
diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts
index 2e34d748..452392f6 100644
--- a/app/ui/dashboard/settings-menus.ts
+++ b/app/ui/dashboard/settings-menus.ts
@@ -13,13 +13,13 @@ import {
openAiApiKeySetting,
enumSettingMenuItem,
firmwareDebugFlagsSetting,
- rawScreenshotsEnabledSetting,
saveVoiceRecordingsSetting,
suspendEvenHubWhenScreenOffSetting,
terminalAuthTokenSetting,
terminalHostSetting,
terminalLaunchPresetsSetting,
terminalPortSetting,
+ terminalWakeOnBellSetting,
textSettingMenuItem,
timeFormatSetting,
toggleSettingMenuItem,
@@ -83,12 +83,12 @@ function settingsSections(): SettingsSection[] {
textSettingMenuItem(terminalPortSetting),
textSettingMenuItem(terminalAuthTokenSetting),
textSettingMenuItem(terminalLaunchPresetsSetting),
+ toggleSettingMenuItem(terminalWakeOnBellSetting),
],
},
{
label: "Developer",
items: [
- toggleSettingMenuItem(rawScreenshotsEnabledSetting),
toggleSettingMenuItem(saveVoiceRecordingsSetting),
toggleSettingMenuItem(firmwareDebugFlagsSetting),
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts
index 0d2a1c28..b910be6e 100644
--- a/app/ui/shell/worker-window.ts
+++ b/app/ui/shell/worker-window.ts
@@ -28,6 +28,15 @@ export type WorkerAppReply =
type: "focus-window";
windowId: string;
}
+ | {
+ /**
+ * Wake the glasses and focus one of the app's windows (e.g. a terminal
+ * bell with wake-on-bell enabled). Ignored unless the screen is off, so
+ * it can never steal focus from an in-use screen.
+ */
+ type: "wake-window";
+ windowId: string;
+ }
| {
/** App-initiated window (e.g. a terminal view opened from the hub list). */
type: "open-window-request";
@@ -149,6 +158,14 @@ export class WorkerAppHost {
this.options.requestShellRender();
}
break;
+ case "wake-window":
+ if (this.openWindows.has(message.windowId) && !shell.isScreenOn()) {
+ // Focus first so wake's foreground-window refresh hits the target.
+ shell.focusWindow(message.windowId);
+ shell.wake("window");
+ this.options.requestShellRender();
+ }
+ break;
case "open-window-request":
this.openWindow({
windowId: message.windowId,
diff --git a/app/workers/terminal-app.worker.ts b/app/workers/terminal-app.worker.ts
index e462e6fe..285d5f57 100644
--- a/app/workers/terminal-app.worker.ts
+++ b/app/workers/terminal-app.worker.ts
@@ -30,6 +30,7 @@ import {
terminalHostSetting,
terminalLaunchPresetsSetting,
terminalPortSetting,
+ terminalWakeOnBellSetting,
} from "../ui/dashboard-settings";
import { TerminalEmulator } from "../ui/apps/terminal-emulator";
import type { DashboardInputEvent } from "../ui/layers";
@@ -76,6 +77,13 @@ type HubWindow = BaseWindow & {
kind: "hub";
selectedIndex: number;
scrollRow: number;
+ /**
+ * Session sockets in display order, captured when the window last became
+ * visible so the list doesn't reshuffle under the user while it's open.
+ * Cleared on foreground/screen-on; orderedSessions() rebuilds it lazily,
+ * sorting by recency and appending sessions that appear while visible.
+ */
+ sessionOrder: string[];
};
type ViewWindow = BaseWindow & {
@@ -179,6 +187,13 @@ let controlClient: G2MirrorClient | null = null;
let controlState: G2MirrorState | null = null;
let controlUnsubscribers: Array<() => void> = [];
+// When each session (by socket) last "updated", unix epoch ms: bells, title
+// changes, and terminal output from open views. Sessions first seen in the
+// initial list after (re)connect are seeded from their last bell, since we
+// weren't watching; ones appearing later were just created, so they get "now".
+const sessionRecency = new Map();
+let controlSessionsSeeded = false;
+
function post(message: WorkerAppReply): void {
global.postMessage(message);
}
@@ -225,6 +240,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
window.foreground = message.foreground;
window.focused = message.focused;
if (window.foreground && window.kind === "view") activeViewId = window.windowId;
+ if (window.foreground && window.kind === "hub") window.sessionOrder = [];
if (window.foreground) renderAndSubmit(window, 0);
break;
}
@@ -232,7 +248,10 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
screenOn = message.on;
if (screenOn) {
for (const window of windows.values()) {
- if (window.foreground) renderAndSubmit(window, 0);
+ if (!window.foreground) continue;
+ // Waking counts as becoming visible: let the session list re-sort.
+ if (window.kind === "hub") window.sessionOrder = [];
+ renderAndSubmit(window, 0);
}
}
break;
@@ -256,8 +275,8 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
// in the dashboard's Settings menu, which lives in the main isolate).
onSettingsStoreChanged((key) => {
if (!key.startsWith("terminal.")) return;
- // The preset list only feeds menus and tool replies; no reconnect needed.
- if (key === "terminal.launchPresets") return;
+ // Settings that don't affect the connection; no reconnect needed.
+ if (key === "terminal.launchPresets" || key === "terminal.wakeOnBell") return;
if (gridCols > 0) {
startControlClient();
}
@@ -287,6 +306,7 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb
menu: null,
selectedIndex: 0,
scrollRow: 0,
+ sessionOrder: [],
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
@@ -333,18 +353,42 @@ function startControlClient(): void {
const client = new G2MirrorClient(options);
controlClient = client;
controlState = client.state();
+ controlSessionsSeeded = false;
controlUnsubscribers.push(
client.onStateChange((state) => {
+ noteSessionListRecency(state);
controlState = state;
renderHubWindows();
}),
- client.onBell((socket) => {
- routeBell(socket);
+ client.onBell((socket, lastBellAtMs) => {
+ routeBell(socket, lastBellAtMs);
+ }),
+ client.onTitle((socket) => {
+ noteSessionUpdated(socket);
}),
);
client.start();
}
+function noteSessionUpdated(socket: string): void {
+ sessionRecency.set(socket, Date.now());
+}
+
+function noteSessionListRecency(state: G2MirrorState): void {
+ for (const session of state.sessions) {
+ const known = sessionRecency.get(session.socket);
+ if (known === undefined) {
+ sessionRecency.set(
+ session.socket,
+ Math.max(session.lastBellAt ?? 0, controlSessionsSeeded ? Date.now() : 0),
+ );
+ } else if ((session.lastBellAt ?? 0) > known) {
+ sessionRecency.set(session.socket, session.lastBellAt!);
+ }
+ }
+ if (state.sessions.length) controlSessionsSeeded = true;
+}
+
function stopControlClient(): void {
for (const unsubscribe of controlUnsubscribers.splice(0)) {
unsubscribe();
@@ -354,12 +398,22 @@ function stopControlClient(): void {
controlState = null;
}
-function routeBell(socket: string): void {
+function routeBell(socket: string, lastBellAtMs: number): void {
+ const known = sessionRecency.get(socket) ?? 0;
+ if (lastBellAtMs > known) sessionRecency.set(socket, lastBellAtMs);
for (const window of windows.values()) {
if (window.kind === "view" && window.socket === socket && !window.foreground) {
post({ type: "set-attention", windowId: window.windowId, attention: true });
}
}
+ if (terminalWakeOnBellSetting.get()) {
+ // Wake to the belling session's view window, or the hub if it has none.
+ // The shell drops the message unless the glasses are actually asleep.
+ const viewId = viewWindowIdForSocket(socket);
+ const hubId = [...windows.values()].find((window) => window.kind === "hub")?.windowId ?? null;
+ const target = viewId ?? hubId;
+ if (target) post({ type: "wake-window", windowId: target });
+ }
}
function renderHubWindows(): void {
@@ -434,6 +488,7 @@ function createViewWindow(
window.emulator.reset();
}
window.receivedData = true;
+ noteSessionUpdated(window.socket);
window.emulator.write(data, () => scheduleRender(window));
}),
client.onSessionDetached((reason) => {
@@ -594,13 +649,36 @@ type HubItem = {
onSelect?: () => void;
};
-function hubItems(): HubItem[] {
+/**
+ * The control connection's sessions in this hub window's display order:
+ * most-recently-updated first as of when the window became visible, with
+ * sessions that appeared since then at the end. The captured order lives in
+ * window.sessionOrder (cleared when the window becomes visible) so the list
+ * doesn't reshuffle while the user is looking at it.
+ */
+function orderedSessions(window: HubWindow): G2MirrorSession[] {
+ const sessions = controlState?.sessions ?? [];
+ const position = new Map();
+ for (const socket of window.sessionOrder) {
+ if (!position.has(socket)) position.set(socket, position.size);
+ }
+ const fresh = sessions
+ .filter((session) => !position.has(session.socket))
+ .sort((a, b) => (sessionRecency.get(b.socket) ?? 0) - (sessionRecency.get(a.socket) ?? 0));
+ for (const session of fresh) {
+ position.set(session.socket, position.size);
+ window.sessionOrder.push(session.socket);
+ }
+ return sessions.slice().sort((a, b) => position.get(a.socket)! - position.get(b.socket)!);
+}
+
+function hubItems(window: HubWindow): HubItem[] {
const items: HubItem[] = [];
const state = controlState;
const phase = state?.phase ?? "idle";
if (phase === "connected" || phase === "attached") {
- for (const session of state?.sessions ?? []) {
+ for (const session of orderedSessions(window)) {
const openWindowId = viewWindowIdForSocket(session.socket);
items.push({
label: openWindowId ? `${sessionLabel(session)} [open]` : sessionLabel(session),
@@ -631,7 +709,7 @@ function hubItems(): HubItem[] {
}
function handleHubInput(window: HubWindow, event: DashboardInputEvent, frameId: number): void {
- const items = hubItems();
+ const items = hubItems(window);
switch (event.type) {
case "scroll-up":
window.selectedIndex = Math.max(0, window.selectedIndex - 1);
@@ -746,7 +824,7 @@ function paintHub(window: HubWindow): GrayImage {
listTop += 28;
}
- const items = hubItems();
+ const items = hubItems(window);
window.selectedIndex = Math.max(0, Math.min(window.selectedIndex, items.length - 1));
const visibleRowCount = Math.max(1, ((viewportHeight - 30 - listTop) / HUB_ROW_HEIGHT) | 0);
window.scrollRow = scrollToKeepSelectionVisible(window.scrollRow, window.selectedIndex, visibleRowCount, items.length);