diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle
index 23cac499..d7622600 100644
--- a/App_Resources/Android/app.gradle
+++ b/App_Resources/Android/app.gradle
@@ -3,6 +3,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
// You can add your native dependencies here
dependencies {
// implementation 'androidx.multidex:multidex:2.0.1'
+ // WebSocket transport for the g2mirror terminal-mirroring client.
+ implementation 'com.squareup.okhttp3:okhttp:4.12.0'
}
def faceclawSherpaOnnxVersion = "1.13.0"
diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml
index 46d9aac6..9ed46c5d 100644
--- a/App_Resources/Android/src/main/AndroidManifest.xml
+++ b/App_Resources/Android/src/main/AndroidManifest.xml
@@ -30,12 +30,16 @@
NameNotFoundException for any package we cannot see. -->
+
{
+ try {
+ listener.onOpen();
+ } catch (Throwable t) {
+ Log.w(TAG, "listener onOpen failed", t);
+ }
+ });
+ }
+
+ @Override public void onMessage(WebSocket webSocket, String text) {
+ mainHandler.post(() -> {
+ try {
+ listener.onTextMessage(text);
+ } catch (Throwable t) {
+ Log.w(TAG, "listener onTextMessage failed", t);
+ }
+ });
+ }
+
+ @Override public void onClosing(WebSocket webSocket, int code, String reason) {
+ webSocket.close(code, reason);
+ }
+
+ @Override public void onClosed(WebSocket webSocket, int code, String reason) {
+ mainHandler.post(() -> {
+ try {
+ listener.onClosed(code, reason == null ? "" : reason);
+ } catch (Throwable t) {
+ Log.w(TAG, "listener onClosed failed", t);
+ }
+ });
+ }
+
+ @Override public void onFailure(WebSocket webSocket, Throwable t, Response response) {
+ if (closeRequested) {
+ return;
+ }
+ final String message = t == null ? "unknown websocket failure" : String.valueOf(t);
+ mainHandler.post(() -> {
+ try {
+ listener.onFailure(message);
+ } catch (Throwable inner) {
+ Log.w(TAG, "listener onFailure failed", inner);
+ }
+ });
+ }
+ });
+ }
+
+ private static OkHttpClient getClient() {
+ OkHttpClient client = sharedClient;
+ if (client == null) {
+ synchronized (FaceclawWebSocket.class) {
+ if (sharedClient == null) {
+ sharedClient = new OkHttpClient.Builder()
+ .pingInterval(20, TimeUnit.SECONDS)
+ .build();
+ }
+ client = sharedClient;
+ }
+ }
+ return client;
+ }
+
+ public boolean sendText(String message) {
+ return socket.send(message == null ? "" : message);
+ }
+
+ public void close(int code, String reason) {
+ closeRequested = true;
+ try {
+ if (!socket.close(code, reason)) {
+ socket.cancel();
+ }
+ } catch (Throwable t) {
+ socket.cancel();
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java
new file mode 100644
index 00000000..f8e8733f
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java
@@ -0,0 +1,9 @@
+package com.faceclaw.app;
+
+/** Callbacks for FaceclawWebSocket, delivered on the Android main thread. */
+public interface FaceclawWebSocketListener {
+ void onOpen();
+ void onTextMessage(String message);
+ void onClosed(int code, String reason);
+ void onFailure(String message);
+}
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java
new file mode 100644
index 00000000..f80cf7ef
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java
@@ -0,0 +1,44 @@
+package com.faceclaw.app;
+
+import android.content.Context;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.ByteBuffer;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+
+/**
+ * Saves raw screenshots for compression experiments. The output is the exact
+ * headerless top-down 4bpp frame buffer that gets zlib-compressed on the wire
+ * (build4bppBmp followed by pack4bppFromBmp), deliberately untransformed — no
+ * resize, no gamma, no header.
+ */
+public final class ScreenshotUtil {
+ private ScreenshotUtil() {}
+
+ /** Returns the absolute path of the written file. */
+ public static String saveRawScreenshot(Context context, ByteBuffer image8bpp, int width, int height) throws java.io.IOException {
+ if (image8bpp == null || width <= 0 || height <= 0 || image8bpp.remaining() < width * height) {
+ throw new IllegalArgumentException("invalid screenshot buffer");
+ }
+ byte[] gray = new byte[image8bpp.remaining()];
+ image8bpp.get(gray);
+ byte[] packed = BmpUtil.pack4bppFromBmp(BmpUtil.build4bppBmp(gray, width, height));
+ if (packed.length == 0) {
+ throw new IllegalStateException("failed to pack 4bpp frame");
+ }
+
+ File dir = new File(context.getExternalFilesDir(null), "screenshots");
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new java.io.IOException("failed to create " + dir);
+ }
+ String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date());
+ File file = new File(dir, "screen-" + stamp + "-" + width + "x" + height + ".4bpp");
+ try (FileOutputStream out = new FileOutputStream(file)) {
+ out.write(packed);
+ }
+ return file.getAbsolutePath();
+ }
+}
diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts
index aa1c1528..ddc0b880 100644
--- a/app/g2/dashboard-controller.ts
+++ b/app/g2/dashboard-controller.ts
@@ -30,10 +30,13 @@ import {
import {
nightscoutApiTokenSetting,
nightscoutSiteUrlSetting,
+ onAnySettingChanged,
+ rawScreenshotsEnabledSetting,
systemCardNameSetting,
voiceControlEnabledSetting,
type ConfigSettingString,
} from "../ui/dashboard-settings";
+import { saveRawScreenshot } from "../native/raw-screenshot";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -49,6 +52,7 @@ export type DashboardSnapshot = {
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
firmwareWarningVisible: boolean;
+ rawScreenshotsEnabled: boolean;
};
type DashboardListener = (snapshot: DashboardSnapshot) => void;
@@ -158,6 +162,11 @@ class DashboardController {
constructor() {
setDashboardActions({
+ requestRender: () => {
+ void this.requestRender("interval").catch((error) => {
+ this.appendLog(`requested render failed: ${this.formatError(error)}`);
+ });
+ },
disconnect: () => this.disconnect(),
startTextSettingEdit: (setting) => this.startTextSettingEdit(setting),
endTextSettingEdit: () => this.endTextSettingEdit(),
@@ -173,6 +182,19 @@ class DashboardController {
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
});
});
+ // Settings toggled from the glasses can change what the phone UI shows
+ // (e.g. the raw-screenshot button), so re-emit the snapshot on any change.
+ onAnySettingChanged(() => this.emit());
+ }
+
+ /**
+ * Save the current dashboard as the raw headerless 4bpp frame buffer the
+ * wire compressor sees; test data for compression experiments.
+ */
+ saveRawDashboardScreenshot(): string {
+ const path = saveRawScreenshot(drawDashboard());
+ this.appendLog(`raw screenshot saved: ${path}`);
+ return path;
}
subscribe(listener: DashboardListener): () => void {
@@ -196,6 +218,7 @@ class DashboardController {
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
firmwareWarningVisible: this.firmwareWarningMessage.length > 0,
+ rawScreenshotsEnabled: rawScreenshotsEnabledSetting.get(),
};
}
diff --git a/app/native/g2mirror-client.ts b/app/native/g2mirror-client.ts
new file mode 100644
index 00000000..28cb0608
--- /dev/null
+++ b/app/native/g2mirror-client.ts
@@ -0,0 +1,320 @@
+import { toUint8Array } from "../util/array-util";
+
+declare const com: any;
+declare const android: any;
+
+/**
+ * Client for the g2mirror terminal-mirroring protocol (see
+ * ../experiments/g2mirror/PROTOCOL.md). One instance per Terminal app
+ * session; talks JSON over a websocket to g2mirror-server, which relays to
+ * one wrapped CLI app at a time.
+ */
+
+const PROTOCOL_VERSION = 1;
+const SESSION_LIST_REFRESH_MS = 3_000;
+
+export type G2MirrorSession = {
+ socket: string;
+ pid: number;
+ cwdHint: string;
+};
+
+export type G2MirrorPhase =
+ | "idle"
+ | "connecting"
+ | "connected" // handshake accepted; can list/attach
+ | "attached" // relaying to one session
+ | "failed";
+
+export type G2MirrorState = {
+ phase: G2MirrorPhase;
+ status: string;
+ sessions: G2MirrorSession[];
+ attachedCommand: string;
+};
+
+export type G2MirrorClientOptions = {
+ host: string;
+ port: number;
+ authToken: string;
+ deviceName: string;
+ cols: number;
+ rows: number;
+};
+
+type TerminalDataKind = "snapshot" | "output";
+
+export class G2MirrorClient {
+ private ws: any = null;
+ private listenerProxy: any = null;
+ private phase: G2MirrorPhase = "idle";
+ private status = "Not connected.";
+ private sessions: G2MirrorSession[] = [];
+ private attachedCommand = "";
+ private listRefreshTimer: ReturnType | null = null;
+ private stopped = false;
+ private readonly stateListeners = new Set<(state: G2MirrorState) => void>();
+ private readonly terminalDataListeners = new Set<(data: Uint8Array, kind: TerminalDataKind) => void>();
+ private readonly sessionAttachedListeners = new Set<(command: string) => void>();
+ private readonly sessionDetachedListeners = new Set<(reason: string) => void>();
+
+ constructor(private readonly options: G2MirrorClientOptions) {}
+
+ state(): G2MirrorState {
+ return {
+ phase: this.phase,
+ status: this.status,
+ sessions: this.sessions.slice(),
+ attachedCommand: this.attachedCommand,
+ };
+ }
+
+ onStateChange(listener: (state: G2MirrorState) => void): () => void {
+ this.stateListeners.add(listener);
+ return () => this.stateListeners.delete(listener);
+ }
+
+ /** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
+ onTerminalData(listener: (data: Uint8Array, kind: TerminalDataKind) => void): () => void {
+ this.terminalDataListeners.add(listener);
+ return () => this.terminalDataListeners.delete(listener);
+ }
+
+ onSessionAttached(listener: (command: string) => void): () => void {
+ this.sessionAttachedListeners.add(listener);
+ return () => this.sessionAttachedListeners.delete(listener);
+ }
+
+ onSessionDetached(listener: (reason: string) => void): () => void {
+ this.sessionDetachedListeners.add(listener);
+ return () => this.sessionDetachedListeners.delete(listener);
+ }
+
+ start(): void {
+ if (this.ws) return;
+ this.stopped = false;
+ const url = `ws://${this.options.host}:${this.options.port}`;
+ this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
+ this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
+ onOpen: () => {
+ if (this.stopped) return;
+ this.setState("connecting", "Authenticating...");
+ this.send({
+ type: "init",
+ version: PROTOCOL_VERSION,
+ auth_token: this.options.authToken,
+ device: this.options.deviceName,
+ width: this.options.cols,
+ height: this.options.rows,
+ });
+ },
+ onTextMessage: (message: string) => {
+ if (this.stopped) return;
+ this.handleMessage(String(message));
+ },
+ onClosed: (code: number, reason: string) => {
+ if (this.stopped) return;
+ this.handleConnectionLost(`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""}).`);
+ },
+ onFailure: (message: string) => {
+ if (this.stopped) return;
+ this.handleConnectionLost(`Connection failed: ${shortenError(String(message))}`);
+ },
+ });
+ try {
+ this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy);
+ } catch (error) {
+ this.ws = null;
+ this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
+ }
+ }
+
+ stop(): void {
+ this.stopped = true;
+ this.clearListRefreshTimer();
+ if (this.ws) {
+ // Best effort: let the wrapped app resize back before we go away.
+ if (this.phase === "attached") {
+ this.send({ type: "unview" });
+ this.send({ type: "disconnect" });
+ }
+ try {
+ this.ws.close(1000, "bye");
+ } catch {
+ // ignore
+ }
+ this.ws = null;
+ }
+ this.listenerProxy = null;
+ this.setState("idle", "Not connected.");
+ }
+
+ listSessions(): void {
+ if (this.phase === "connected" || this.phase === "attached") {
+ this.send({ type: "list" });
+ }
+ }
+
+ connectSession(socket: string): void {
+ if (this.phase !== "connected") return;
+ this.setState(this.phase, "Attaching to session...");
+ this.send({ type: "connect", socket });
+ }
+
+ disconnectSession(): void {
+ if (this.phase !== "attached") return;
+ this.send({ type: "unview" });
+ this.send({ type: "disconnect" });
+ }
+
+ view(): void {
+ if (this.phase !== "attached") return;
+ this.send({ type: "view" });
+ }
+
+ unview(): void {
+ if (this.phase !== "attached") return;
+ this.send({ type: "unview" });
+ }
+
+ private handleMessage(text: string): void {
+ let message: any;
+ try {
+ message = JSON.parse(text);
+ } catch {
+ return;
+ }
+ if (!message || typeof message.type !== "string") return;
+
+ switch (message.type) {
+ case "init":
+ this.setState("connected", "Connected.");
+ this.listSessions();
+ this.ensureListRefreshTimer();
+ return;
+ case "error": {
+ const errorText = String(message.message ?? "unknown error");
+ if (this.phase === "connecting") {
+ // Handshake rejection; the server closes the socket after this.
+ this.setState("failed", `Rejected: ${errorText}`);
+ } else {
+ this.setState(this.phase, `Server error: ${errorText}`);
+ }
+ return;
+ }
+ case "sessions": {
+ const raw = Array.isArray(message.sessions) ? message.sessions : [];
+ this.sessions = raw
+ .map((item: any): G2MirrorSession => ({
+ socket: String(item?.socket ?? ""),
+ pid: Number(item?.pid) || 0,
+ cwdHint: String(item?.cwd_hint ?? ""),
+ }))
+ .filter((session: G2MirrorSession) => session.socket.length > 0);
+ this.emitState();
+ return;
+ }
+ case "connect": {
+ // Session accepted us (relayed from the wrapper).
+ this.clearListRefreshTimer();
+ this.attachedCommand = String(message.command ?? "");
+ this.setState("attached", `Attached: ${this.attachedCommand || "session"}`);
+ for (const listener of Array.from(this.sessionAttachedListeners)) {
+ listener(this.attachedCommand);
+ }
+ return;
+ }
+ case "snapshot":
+ case "output": {
+ const data = decodeBase64(String(message.data ?? ""));
+ for (const listener of Array.from(this.terminalDataListeners)) {
+ listener(data, message.type as TerminalDataKind);
+ }
+ return;
+ }
+ case "exit": {
+ const status = message.status === null || message.status === undefined ? "signal" : String(message.status);
+ this.setState(this.phase, `Session exited (status ${status}).`);
+ return;
+ }
+ case "disconnected": {
+ const reason = String(message.reason ?? "unknown");
+ this.attachedCommand = "";
+ if (this.phase === "attached") {
+ this.setState("connected", `Detached (${reason}).`);
+ this.ensureListRefreshTimer();
+ this.listSessions();
+ for (const listener of Array.from(this.sessionDetachedListeners)) {
+ listener(reason);
+ }
+ }
+ return;
+ }
+ default:
+ // Unknown message types are expected; ignore for forward compatibility.
+ return;
+ }
+ }
+
+ private handleConnectionLost(statusText: string): void {
+ this.clearListRefreshTimer();
+ this.ws = null;
+ this.listenerProxy = null;
+ const wasAttached = this.phase === "attached";
+ this.attachedCommand = "";
+ this.setState("failed", statusText);
+ if (wasAttached) {
+ for (const listener of Array.from(this.sessionDetachedListeners)) {
+ listener("connection lost");
+ }
+ }
+ }
+
+ private ensureListRefreshTimer(): void {
+ if (this.listRefreshTimer) return;
+ this.listRefreshTimer = setInterval(() => {
+ if (this.phase === "connected") {
+ this.listSessions();
+ }
+ }, SESSION_LIST_REFRESH_MS);
+ }
+
+ private clearListRefreshTimer(): void {
+ if (this.listRefreshTimer) {
+ clearInterval(this.listRefreshTimer);
+ this.listRefreshTimer = null;
+ }
+ }
+
+ private send(message: object): void {
+ if (!this.ws) return;
+ try {
+ this.ws.sendText(JSON.stringify(message));
+ } catch (error) {
+ console.warn("g2mirror send failed", error);
+ }
+ }
+
+ private setState(phase: G2MirrorPhase, status: string): void {
+ this.phase = phase;
+ this.status = status;
+ this.emitState();
+ }
+
+ private emitState(): void {
+ const state = this.state();
+ for (const listener of Array.from(this.stateListeners)) {
+ listener(state);
+ }
+ }
+}
+
+function decodeBase64(data: string): Uint8Array {
+ if (!data) return new Uint8Array(0);
+ return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT));
+}
+
+function shortenError(message: string): string {
+ const collapsed = message.replace(/\s+/g, " ").trim();
+ return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`;
+}
diff --git a/app/native/raw-screenshot.ts b/app/native/raw-screenshot.ts
new file mode 100644
index 00000000..39e0ebc4
--- /dev/null
+++ b/app/native/raw-screenshot.ts
@@ -0,0 +1,24 @@
+import { Utils } from "@nativescript/core";
+
+import { type GrayImage } from "../graphics/image";
+
+declare const com: any;
+
+/**
+ * Save an image as the raw headerless 576x288 4bpp frame buffer (the exact
+ * bytes the wire compressor sees) for compression experiments. Returns the
+ * absolute path of the written file; retrieve with
+ * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/
+ */
+export function saveRawScreenshot(image: GrayImage): string {
+ const context = Utils.android.getApplicationContext();
+ if (!context) throw new Error("Android application context unavailable");
+ return String(
+ com.faceclaw.app.ScreenshotUtil.saveRawScreenshot(
+ context,
+ image.pixels.buffer,
+ image.width,
+ image.height,
+ ),
+ );
+}
diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml
index 72459a36..99311679 100644
--- a/app/phone-ui/main-page.xml
+++ b/app/phone-ui/main-page.xml
@@ -25,6 +25,7 @@
+
@@ -49,12 +50,15 @@
width="{{ landscapeDisplayPreviewWidth }}"
height="{{ landscapeDisplayPreviewHeight }}" />
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts
index 8e7a341a..fe02aa21 100644
--- a/app/phone-ui/main-view-model.ts
+++ b/app/phone-ui/main-view-model.ts
@@ -15,6 +15,7 @@ export class MainViewModel extends Observable {
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
private _firmwareWarningVisible = false;
+ private _rawScreenshotsEnabled = false;
private _showLog = false;
private _phase: "disconnected" | "connecting" | "connected" | "disconnecting" = "disconnected";
@@ -32,6 +33,7 @@ export class MainViewModel extends Observable {
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
this.firmwareWarningVisible = snapshot.firmwareWarningVisible;
+ this.rawScreenshotsEnabled = snapshot.rawScreenshotsEnabled;
});
}
@@ -230,6 +232,30 @@ export class MainViewModel extends Observable {
return this._firmwareWarningVisible ? "visible" : "collapse";
}
+ get rawScreenshotsEnabled(): boolean {
+ return this._rawScreenshotsEnabled;
+ }
+
+ set rawScreenshotsEnabled(value: boolean) {
+ if (this._rawScreenshotsEnabled !== value) {
+ this._rawScreenshotsEnabled = value;
+ this.notifyPropertyChange("rawScreenshotsEnabled", value);
+ this.notifyPropertyChange("screenshotButtonVisibility", this.screenshotButtonVisibility);
+ }
+ }
+
+ get screenshotButtonVisibility(): "visible" | "collapse" {
+ return this._rawScreenshotsEnabled ? "visible" : "collapse";
+ }
+
+ onScreenshotTap(): void {
+ try {
+ dashboardController.saveRawDashboardScreenshot();
+ } catch (error) {
+ console.error("raw screenshot failed", error);
+ }
+ }
+
get phase(): "disconnected" | "connecting" | "connected" | "disconnecting" {
return this._phase;
}
diff --git a/app/ui/apps/terminal-emulator.ts b/app/ui/apps/terminal-emulator.ts
new file mode 100644
index 00000000..b08a3f63
--- /dev/null
+++ b/app/ui/apps/terminal-emulator.ts
@@ -0,0 +1,73 @@
+import type { Terminal as XtermTerminal } from "@xterm/headless";
+
+// xterm-headless expects a few browser globals (navigator for platform
+// sniffing, window for timer/scheduling lookups) that the NativeScript
+// runtime doesn't define. Provide minimal stand-ins before the library is
+// evaluated (hence require, not a hoisted import).
+const runtimeGlobal = globalThis as any;
+if (typeof runtimeGlobal.navigator === "undefined") {
+ runtimeGlobal.navigator = {
+ userAgent: "NativeScript",
+ platform: "Android",
+ language: "en",
+ languages: ["en"],
+ };
+}
+if (typeof runtimeGlobal.window === "undefined") {
+ runtimeGlobal.window = runtimeGlobal;
+}
+if (typeof runtimeGlobal.self === "undefined") {
+ runtimeGlobal.self = runtimeGlobal;
+}
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { Terminal } = require("@xterm/headless") as typeof import("@xterm/headless");
+
+/**
+ * Headless terminal emulator for g2mirror streams: feed it the raw
+ * VT100/xterm bytes from snapshot/output messages and read back the visible
+ * text grid. Formatting (color/bold/inverse) is deliberately dropped for
+ * now — we only mirror the text.
+ */
+export class TerminalEmulator {
+ private readonly term: XtermTerminal;
+
+ constructor(
+ readonly cols: number,
+ readonly rows: number,
+ ) {
+ this.term = new Terminal({
+ cols,
+ rows,
+ scrollback: 0,
+ allowProposedApi: true,
+ });
+ }
+
+ /**
+ * Apply terminal bytes. onProcessed fires once the emulator has actually
+ * parsed them (xterm buffers writes internally), so schedule repaints there.
+ */
+ write(data: Uint8Array, onProcessed: () => void): void {
+ this.term.write(data, onProcessed);
+ }
+
+ /** Full reset; snapshots are defined as repainting from a cleared screen. */
+ reset(): void {
+ this.term.reset();
+ }
+
+ visibleLines(): string[] {
+ const buffer = this.term.buffer.active;
+ const lines: string[] = [];
+ for (let row = 0; row < this.rows; row++) {
+ const line = buffer.getLine(buffer.viewportY + row);
+ lines.push(line ? line.translateToString(true) : "");
+ }
+ return lines;
+ }
+
+ cursor(): { x: number; y: number } {
+ const buffer = this.term.buffer.active;
+ return { x: buffer.cursorX, y: buffer.cursorY };
+ }
+}
diff --git a/app/ui/apps/terminal.ts b/app/ui/apps/terminal.ts
new file mode 100644
index 00000000..78dedf35
--- /dev/null
+++ b/app/ui/apps/terminal.ts
@@ -0,0 +1,316 @@
+import { getFont } from "~/graphics/bdffont";
+import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "~/graphics/image";
+import {
+ G2MirrorClient,
+ type G2MirrorSession,
+ type G2MirrorState,
+} from "~/native/g2mirror-client";
+import {
+ terminalAuthTokenSetting,
+ terminalHostSetting,
+ terminalPortSetting,
+ textSettingMenuItem,
+} from "../dashboard-settings";
+import { TOP_LEFT_MENU_LAYOUT } from "../dashboard";
+import { MenuLayer } from "../menu";
+import { type DashboardInputEvent, type Layer, type LayerActions, type LayerContext } from "../layers";
+
+// Terminus-12 has a 6x12 cell, so the full 576x288 lens is a 96x24 grid —
+// this is the size we declare in the g2mirror init handshake, fixed per
+// websocket connection.
+const terminalFont = getFont("terminus12");
+const CELL_WIDTH = 6;
+const CELL_HEIGHT = 12;
+const TERMINAL_COLS = G2_LENS_WIDTH / CELL_WIDTH;
+const TERMINAL_ROWS = G2_LENS_HEIGHT / CELL_HEIGHT;
+const DEVICE_NAME = "Faceclaw G2";
+
+const HUB_ROW_HEIGHT = 20;
+const HUB_LIST_TOP = 64;
+
+type HubItem = {
+ label: string;
+ onSelect: (ctx: LayerContext) => void;
+};
+
+/**
+ * The Terminal app hub: connection status, list of live g2mirror sessions,
+ * and the app-local settings menu. Selecting a session attaches and opens
+ * the full-screen readonly terminal view.
+ */
+export class TerminalAppLayer implements Layer {
+ private client: G2MirrorClient | null = null;
+ private clientState: G2MirrorState | null = null;
+ private selectedIndex = 0;
+ private lastCtx: LayerContext | null = null;
+ private activeViewLayer: TerminalViewLayer | null = null;
+ private unsubscribers: Array<() => void> = [];
+
+ constructor(private readonly actions: LayerActions) {
+ selfTestEmulator();
+ this.startClient();
+ }
+
+ private startClient(): void {
+ this.stopClient();
+ const host = terminalHostSetting.get().trim();
+ if (!host) return;
+ const port = parseInt(terminalPortSetting.get(), 10) || 8737;
+ const client = new G2MirrorClient({
+ host,
+ port,
+ authToken: terminalAuthTokenSetting.get(),
+ deviceName: DEVICE_NAME,
+ cols: TERMINAL_COLS,
+ rows: TERMINAL_ROWS,
+ });
+ this.client = client;
+ this.clientState = client.state();
+ this.unsubscribers.push(
+ client.onStateChange((state) => {
+ this.clientState = state;
+ this.actions.requestRender();
+ }),
+ client.onSessionAttached(() => {
+ this.openViewLayer();
+ }),
+ client.onSessionDetached(() => {
+ this.closeViewLayer();
+ }),
+ );
+ client.start();
+ }
+
+ private stopClient(): void {
+ for (const unsubscribe of this.unsubscribers.splice(0)) {
+ unsubscribe();
+ }
+ this.client?.stop();
+ this.client = null;
+ this.clientState = null;
+ }
+
+ private openViewLayer(): void {
+ const ctx = this.lastCtx;
+ if (!ctx || !this.client || this.activeViewLayer) return;
+ const viewLayer = new TerminalViewLayer(this.client, this.actions, () => {
+ if (this.activeViewLayer === viewLayer) {
+ this.activeViewLayer = null;
+ }
+ });
+ this.activeViewLayer = viewLayer;
+ ctx.stack.push(viewLayer);
+ this.actions.requestRender();
+ }
+
+ private closeViewLayer(): void {
+ const ctx = this.lastCtx;
+ const viewLayer = this.activeViewLayer;
+ if (!ctx || !viewLayer) return;
+ ctx.stack.popIfTop((layer) => layer === viewLayer);
+ this.actions.requestRender();
+ }
+
+ paint(ctx: LayerContext): GrayImage {
+ this.lastCtx = ctx;
+ const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
+ image.drawText(terminalFont, 18, 14, "Terminal", 220);
+ image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
+ image.drawText(terminalFont, 24, 38, this.statusLine(), 170);
+
+ let listTop = HUB_LIST_TOP;
+ if (!terminalHostSetting.get().trim()) {
+ image.drawText(terminalFont, 24, 56, "To set up a server, see:", 150);
+ image.drawText(terminalFont, 24, 70, "https://github.com/jimrandomh/g2mirror", 190);
+ listTop += 32;
+ }
+
+ const items = this.hubItems();
+ this.selectedIndex = Math.max(0, Math.min(this.selectedIndex, items.length - 1));
+ for (let index = 0; index < items.length; index++) {
+ const y = listTop + index * HUB_ROW_HEIGHT;
+ if (y + HUB_ROW_HEIGHT > G2_LENS_HEIGHT - 40) break;
+ const selected = index === this.selectedIndex;
+ if (selected) {
+ image.fillRoundedRect(20, y - 2, G2_LENS_WIDTH - 40, HUB_ROW_HEIGHT - 1, 15);
+ image.drawRoundedRect(20, y - 2, G2_LENS_WIDTH - 40, HUB_ROW_HEIGHT - 1, 45);
+ }
+ image.drawText(terminalFont, 32, y + 2, items[index]!.label, selected ? 255 : 200);
+ }
+
+ image.drawText(terminalFont, 24, G2_LENS_HEIGHT - 36, "Double-click to go back", 110);
+ return image;
+ }
+
+ async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise {
+ this.lastCtx = ctx;
+ const items = this.hubItems();
+ switch (event.type) {
+ case "double-click":
+ ctx.stack.pop();
+ return;
+ case "scroll-up":
+ this.selectedIndex = Math.max(0, this.selectedIndex - 1);
+ return;
+ case "scroll-down":
+ this.selectedIndex = Math.min(items.length - 1, this.selectedIndex + 1);
+ return;
+ case "click": {
+ const item = items[Math.max(0, Math.min(this.selectedIndex, items.length - 1))];
+ item?.onSelect(ctx);
+ return;
+ }
+ default:
+ return;
+ }
+ }
+
+ onRemoved(): void {
+ this.stopClient();
+ }
+
+ private statusLine(): string {
+ const host = terminalHostSetting.get().trim();
+ if (!host) {
+ return "No host configured. Open Settings below.";
+ }
+ return this.clientState?.status ?? "Not connected.";
+ }
+
+ private hubItems(): HubItem[] {
+ const items: HubItem[] = [];
+ const state = this.clientState;
+ const phase = state?.phase ?? "idle";
+
+ if (phase === "connected" || phase === "attached") {
+ for (const session of state?.sessions ?? []) {
+ items.push({
+ label: sessionLabel(session),
+ onSelect: () => {
+ this.client?.connectSession(session.socket);
+ },
+ });
+ }
+ if (!state?.sessions.length) {
+ items.push({
+ label: "(no live sessions; run g2mirror )",
+ onSelect: () => {
+ this.client?.listSessions();
+ },
+ });
+ }
+ }
+ if (phase === "idle" || phase === "failed") {
+ items.push({
+ label: terminalHostSetting.get().trim() ? "Connect" : "Connect (host not set)",
+ onSelect: () => {
+ this.startClient();
+ },
+ });
+ }
+ items.push({
+ label: "Settings",
+ onSelect: (ctx) => {
+ ctx.stack.push(createTerminalSettingsMenuLayer());
+ },
+ });
+ return items;
+ }
+}
+
+/** Full-screen readonly mirror of the attached session. */
+class TerminalViewLayer implements Layer {
+ private readonly emulator: import("./terminal-emulator").TerminalEmulator;
+ private receivedData = false;
+ private unsubscribeData: (() => void) | null = null;
+
+ constructor(
+ private readonly client: G2MirrorClient,
+ private readonly actions: LayerActions,
+ private readonly onClosed: () => void,
+ ) {
+ // Lazy import keeps xterm out of the startup path; it is only needed
+ // once a terminal is actually viewed.
+ const { TerminalEmulator } = require("./terminal-emulator") as typeof import("./terminal-emulator");
+ this.emulator = new TerminalEmulator(TERMINAL_COLS, TERMINAL_ROWS);
+ this.unsubscribeData = client.onTerminalData((data, kind) => {
+ if (kind === "snapshot") {
+ this.emulator.reset();
+ }
+ this.receivedData = true;
+ this.emulator.write(data, () => this.actions.requestRender());
+ });
+ client.view();
+ }
+
+ paint(): GrayImage {
+ const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
+ if (!this.receivedData) {
+ image.drawText(terminalFont, 24, 130, "Waiting for terminal output...", 170);
+ image.drawText(terminalFont, 24, 150, "Double-click to go back", 110);
+ return image;
+ }
+ const cursor = this.emulator.cursor();
+ image.fillRect(cursor.x * CELL_WIDTH, cursor.y * CELL_HEIGHT, CELL_WIDTH, CELL_HEIGHT, 70);
+ const lines = this.emulator.visibleLines();
+ for (let row = 0; row < lines.length; row++) {
+ const line = lines[row]!;
+ if (line.length === 0) continue;
+ image.drawText(terminalFont, 0, row * CELL_HEIGHT, line, 200);
+ }
+ return image;
+ }
+
+ handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
+ // Readonly for now: protocol v1 has no keyboard input, so scrolls and
+ // clicks are ignored rather than forwarded.
+ if (event.type === "double-click") {
+ ctx.stack.pop();
+ }
+ }
+
+ onRemoved(): void {
+ this.unsubscribeData?.();
+ this.unsubscribeData = null;
+ // Let the wrapped app resize back for the person at the real keyboard,
+ // and free the session for other clients.
+ this.client.unview();
+ this.client.disconnectSession();
+ this.onClosed();
+ }
+}
+
+// One-time check that the xterm-headless emulator evaluates and parses in
+// this JS runtime, run at app open so a broken dependency surfaces in the log
+// immediately instead of mid-session-attach.
+let emulatorSelfTested = false;
+function selfTestEmulator(): void {
+ if (emulatorSelfTested) return;
+ emulatorSelfTested = true;
+ try {
+ const { TerminalEmulator } = require("./terminal-emulator") as typeof import("./terminal-emulator");
+ const emulator = new TerminalEmulator(8, 2);
+ emulator.write(Uint8Array.from([0x68, 0x69, 0x1b, 0x5b, 0x31, 0x6d, 0x21]), () => {
+ console.log(`terminal emulator self-test: ${JSON.stringify(emulator.visibleLines())}`);
+ });
+ } catch (error) {
+ console.error("terminal emulator failed to load:", error);
+ }
+}
+
+function createTerminalSettingsMenuLayer(): MenuLayer {
+ return new MenuLayer(
+ "Terminal > Settings",
+ [
+ textSettingMenuItem(terminalHostSetting),
+ textSettingMenuItem(terminalPortSetting),
+ textSettingMenuItem(terminalAuthTokenSetting),
+ ],
+ TOP_LEFT_MENU_LAYOUT,
+ );
+}
+
+function sessionLabel(session: G2MirrorSession): string {
+ const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/");
+ return `${hint || "session"} (pid ${session.pid})`;
+}
diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts
index 56049958..9b134a4b 100644
--- a/app/ui/dashboard-settings.ts
+++ b/app/ui/dashboard-settings.ts
@@ -34,6 +34,24 @@ type ConfigSettingOptions = {
formatValue?: (value: TValue) => string;
};
+// Fired after any setting's set(); lets phone-side UI that depends on
+// settings toggled from the glasses (e.g. the raw-screenshot button) update
+// without waiting for an unrelated snapshot emit.
+const settingChangeListeners = new Set<() => void>();
+
+export function onAnySettingChanged(listener: () => void): () => void {
+ settingChangeListeners.add(listener);
+ return () => {
+ settingChangeListeners.delete(listener);
+ };
+}
+
+function notifySettingChanged(): void {
+ for (const listener of Array.from(settingChangeListeners)) {
+ listener();
+ }
+}
+
export abstract class ConfigSetting {
readonly id: TId;
readonly label: string;
@@ -69,6 +87,7 @@ export class ConfigSettingBoolean extends ConfigSet
set(value: boolean): boolean {
ApplicationSettings.setBoolean(this.storageKey, value);
+ notifySettingChanged();
return value;
}
@@ -107,6 +126,7 @@ export class ConfigSettingEnum extends ConfigSett
set(value: string): string {
const normalized = this.normalizer(value);
ApplicationSettings.setString(this.storageKey, normalized);
+ notifySettingChanged();
return normalized;
}
}
@@ -239,6 +260,40 @@ export const voiceControlEnabledSetting = new ConfigSettingBoolean({
defaultValue: false,
});
+export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
+ id: "raw-screenshots-enabled",
+ label: "Take raw screenshots",
+ storageKey: "developer.rawScreenshots",
+ defaultValue: false,
+});
+
+export const terminalHostSetting = new ConfigSettingString({
+ id: "terminal-host",
+ label: "Host",
+ storageKey: "terminal.host",
+ defaultValue: "",
+ editorTitle: "g2mirror host (tailscale IP)",
+ glassesEditTitle: "Edit terminal host",
+});
+
+export const terminalPortSetting = new ConfigSettingString({
+ id: "terminal-port",
+ label: "Port",
+ storageKey: "terminal.port",
+ defaultValue: "8737",
+ editorTitle: "g2mirror port",
+ glassesEditTitle: "Edit terminal port",
+});
+
+export const terminalAuthTokenSetting = new ConfigSettingString({
+ id: "terminal-auth-token",
+ label: "Auth token",
+ storageKey: "terminal.authToken",
+ defaultValue: "",
+ editorTitle: "g2mirror auth token",
+ glassesEditTitle: "Edit terminal auth token",
+});
+
export const nightscoutSiteUrlSetting = new ConfigSettingString({
id: "nightscout-site-url",
label: "Nightscout site URL",
diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts
index f44d4c8f..cfa2d239 100644
--- a/app/ui/dashboard.ts
+++ b/app/ui/dashboard.ts
@@ -40,6 +40,7 @@ export let dashboardState: DashboardState = {
};
const dashboardActions: LayerActions = {
+ requestRender: () => {},
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
diff --git a/app/ui/dashboard/root-menu.ts b/app/ui/dashboard/root-menu.ts
index 84a9fd2b..a3ea3fd5 100644
--- a/app/ui/dashboard/root-menu.ts
+++ b/app/ui/dashboard/root-menu.ts
@@ -7,6 +7,7 @@ import { MenuLayer } from "../menu";
import { NotificationsListLayer } from "../notifications";
import { createNightscoutSettingsMenuLayer, createSettingsMenuLayer } from "./settings-menus";
import { TelepromptLayer } from "../apps/teleprompt";
+import { TerminalAppLayer } from "../apps/terminal";
import { StopwatchLayer } from "../apps/stopwatch";
import { TranscribeLayer } from "../apps/transcribe";
import { ScreenTestLayer } from "../apps/screen-test";
@@ -100,6 +101,12 @@ function createAppsMenuLayer(): MenuLayer {
layer.start(ctx);
},
},
+ {
+ label: "Terminal",
+ onSelect: (ctx) => {
+ ctx.stack.push(new TerminalAppLayer(ctx.actions));
+ },
+ },
{
label: "Screen test",
onSelect: (ctx) => {
diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts
index da2b0b37..3b862ac9 100644
--- a/app/ui/dashboard/settings-menus.ts
+++ b/app/ui/dashboard/settings-menus.ts
@@ -1,5 +1,5 @@
import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
-import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
+import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { MenuLayer } from "../menu";
export function createSettingsMenuLayer(): MenuLayer {
@@ -30,6 +30,22 @@ export function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createIntegrationsMenuLayer());
},
},
+ {
+ label: "Developer",
+ onSelect: (ctx) => {
+ ctx.stack.push(createDeveloperSettingsMenuLayer());
+ },
+ },
+ ],
+ TOP_LEFT_MENU_LAYOUT,
+ );
+}
+
+function createDeveloperSettingsMenuLayer(): MenuLayer {
+ return new MenuLayer(
+ "Settings > Developer",
+ [
+ toggleSettingMenuItem(rawScreenshotsEnabledSetting),
],
TOP_LEFT_MENU_LAYOUT,
);
diff --git a/app/ui/layers.ts b/app/ui/layers.ts
index 0ad1d745..f9c913f8 100644
--- a/app/ui/layers.ts
+++ b/app/ui/layers.ts
@@ -11,6 +11,8 @@ export type DashboardInputEvent =
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
export type LayerActions = {
+ /** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
+ requestRender: () => void;
disconnect: () => Promise | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise | void;
endTextSettingEdit: () => Promise | void;
@@ -24,6 +26,14 @@ export type LayerActions = {
export type PaintBelow = () => GrayImage;
+function notifyRemoved(layer: Layer | undefined): void {
+ try {
+ layer?.onRemoved?.();
+ } catch (error) {
+ console.warn("layer onRemoved failed", error);
+ }
+}
+
export interface LayerContext {
readonly stack: LayerStack;
readonly actions: LayerActions;
@@ -33,6 +43,8 @@ export interface Layer {
readonly paintOverBase?: boolean;
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage;
handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise | void;
+ /** Called when the layer leaves the stack by any path (pop or clearToBase). */
+ onRemoved?(): void;
}
export class LayerStack {
@@ -53,21 +65,23 @@ export class LayerStack {
pop(): void {
if (this.layers.length > 1) {
- this.layers.pop();
+ notifyRemoved(this.layers.pop());
}
}
/** Pop the top layer only if it matches; returns whether a layer was popped. */
popIfTop(predicate: (layer: Layer) => boolean): boolean {
if (this.layers.length > 1 && predicate(this.layers[this.layers.length - 1]!)) {
- this.layers.pop();
+ notifyRemoved(this.layers.pop());
return true;
}
return false;
}
clearToBase(): void {
- this.layers.splice(1);
+ for (const layer of this.layers.splice(1)) {
+ notifyRemoved(layer);
+ }
}
isAtBase(): boolean {
diff --git a/package-lock.json b/package-lock.json
index 87e6185a..f99a0212 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,7 @@
"dependencies": {
"@nativescript/core": "~9.0.0",
"@nativescript/theme": "^3.1.0",
+ "@xterm/headless": "^6.0.0",
"lodash": "^4.18.1",
"lz4js": "^0.2.0",
"pako": "^2.1.0",
@@ -1288,6 +1289,15 @@
}
}
},
+ "node_modules/@xterm/headless": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/@xterm/headless/-/headless-6.0.0.tgz",
+ "integrity": "sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw==",
+ "license": "MIT",
+ "workspaces": [
+ "addons/*"
+ ]
+ },
"node_modules/@xtuc/ieee754": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
diff --git a/package.json b/package.json
index 720dde31..4d487b93 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,7 @@
"dependencies": {
"@nativescript/core": "~9.0.0",
"@nativescript/theme": "^3.1.0",
+ "@xterm/headless": "^6.0.0",
"lodash": "^4.18.1",
"lz4js": "^0.2.0",
"pako": "^2.1.0",