diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 23cac499..d7622600 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -3,6 +3,8 @@ import org.apache.tools.ant.taskdefs.condition.Os // You can add your native dependencies here dependencies { // implementation 'androidx.multidex:multidex:2.0.1' + // WebSocket transport for the g2mirror terminal-mirroring client. + implementation 'com.squareup.okhttp3:okhttp:4.12.0' } def faceclawSherpaOnnxVersion = "1.13.0" diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml index 46d9aac6..9ed46c5d 100644 --- a/App_Resources/Android/src/main/AndroidManifest.xml +++ b/App_Resources/Android/src/main/AndroidManifest.xml @@ -30,12 +30,16 @@ NameNotFoundException for any package we cannot see. --> + { + try { + listener.onOpen(); + } catch (Throwable t) { + Log.w(TAG, "listener onOpen failed", t); + } + }); + } + + @Override public void onMessage(WebSocket webSocket, String text) { + mainHandler.post(() -> { + try { + listener.onTextMessage(text); + } catch (Throwable t) { + Log.w(TAG, "listener onTextMessage failed", t); + } + }); + } + + @Override public void onClosing(WebSocket webSocket, int code, String reason) { + webSocket.close(code, reason); + } + + @Override public void onClosed(WebSocket webSocket, int code, String reason) { + mainHandler.post(() -> { + try { + listener.onClosed(code, reason == null ? "" : reason); + } catch (Throwable t) { + Log.w(TAG, "listener onClosed failed", t); + } + }); + } + + @Override public void onFailure(WebSocket webSocket, Throwable t, Response response) { + if (closeRequested) { + return; + } + final String message = t == null ? "unknown websocket failure" : String.valueOf(t); + mainHandler.post(() -> { + try { + listener.onFailure(message); + } catch (Throwable inner) { + Log.w(TAG, "listener onFailure failed", inner); + } + }); + } + }); + } + + private static OkHttpClient getClient() { + OkHttpClient client = sharedClient; + if (client == null) { + synchronized (FaceclawWebSocket.class) { + if (sharedClient == null) { + sharedClient = new OkHttpClient.Builder() + .pingInterval(20, TimeUnit.SECONDS) + .build(); + } + client = sharedClient; + } + } + return client; + } + + public boolean sendText(String message) { + return socket.send(message == null ? "" : message); + } + + public void close(int code, String reason) { + closeRequested = true; + try { + if (!socket.close(code, reason)) { + socket.cancel(); + } + } catch (Throwable t) { + socket.cancel(); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java new file mode 100644 index 00000000..f8e8733f --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWebSocketListener.java @@ -0,0 +1,9 @@ +package com.faceclaw.app; + +/** Callbacks for FaceclawWebSocket, delivered on the Android main thread. */ +public interface FaceclawWebSocketListener { + void onOpen(); + void onTextMessage(String message); + void onClosed(int code, String reason); + void onFailure(String message); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java new file mode 100644 index 00000000..f80cf7ef --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ScreenshotUtil.java @@ -0,0 +1,44 @@ +package com.faceclaw.app; + +import android.content.Context; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; + +/** + * Saves raw screenshots for compression experiments. The output is the exact + * headerless top-down 4bpp frame buffer that gets zlib-compressed on the wire + * (build4bppBmp followed by pack4bppFromBmp), deliberately untransformed — no + * resize, no gamma, no header. + */ +public final class ScreenshotUtil { + private ScreenshotUtil() {} + + /** Returns the absolute path of the written file. */ + public static String saveRawScreenshot(Context context, ByteBuffer image8bpp, int width, int height) throws java.io.IOException { + if (image8bpp == null || width <= 0 || height <= 0 || image8bpp.remaining() < width * height) { + throw new IllegalArgumentException("invalid screenshot buffer"); + } + byte[] gray = new byte[image8bpp.remaining()]; + image8bpp.get(gray); + byte[] packed = BmpUtil.pack4bppFromBmp(BmpUtil.build4bppBmp(gray, width, height)); + if (packed.length == 0) { + throw new IllegalStateException("failed to pack 4bpp frame"); + } + + File dir = new File(context.getExternalFilesDir(null), "screenshots"); + if (!dir.exists() && !dir.mkdirs()) { + throw new java.io.IOException("failed to create " + dir); + } + String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date()); + File file = new File(dir, "screen-" + stamp + "-" + width + "x" + height + ".4bpp"); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(packed); + } + return file.getAbsolutePath(); + } +} diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index aa1c1528..ddc0b880 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -30,10 +30,13 @@ import { import { nightscoutApiTokenSetting, nightscoutSiteUrlSetting, + onAnySettingChanged, + rawScreenshotsEnabledSetting, systemCardNameSetting, voiceControlEnabledSetting, type ConfigSettingString, } from "../ui/dashboard-settings"; +import { saveRawScreenshot } from "../native/raw-screenshot"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting"; @@ -49,6 +52,7 @@ export type DashboardSnapshot = { evenAppConflictWarningVisible: boolean; firmwareWarningMessage: string; firmwareWarningVisible: boolean; + rawScreenshotsEnabled: boolean; }; type DashboardListener = (snapshot: DashboardSnapshot) => void; @@ -158,6 +162,11 @@ class DashboardController { constructor() { setDashboardActions({ + requestRender: () => { + void this.requestRender("interval").catch((error) => { + this.appendLog(`requested render failed: ${this.formatError(error)}`); + }); + }, disconnect: () => this.disconnect(), startTextSettingEdit: (setting) => this.startTextSettingEdit(setting), endTextSettingEdit: () => this.endTextSettingEdit(), @@ -173,6 +182,19 @@ class DashboardController { this.appendLog(`notification wake failed: ${this.formatError(error)}`); }); }); + // Settings toggled from the glasses can change what the phone UI shows + // (e.g. the raw-screenshot button), so re-emit the snapshot on any change. + onAnySettingChanged(() => this.emit()); + } + + /** + * Save the current dashboard as the raw headerless 4bpp frame buffer the + * wire compressor sees; test data for compression experiments. + */ + saveRawDashboardScreenshot(): string { + const path = saveRawScreenshot(drawDashboard()); + this.appendLog(`raw screenshot saved: ${path}`); + return path; } subscribe(listener: DashboardListener): () => void { @@ -196,6 +218,7 @@ class DashboardController { evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0, firmwareWarningMessage: this.firmwareWarningMessage, firmwareWarningVisible: this.firmwareWarningMessage.length > 0, + rawScreenshotsEnabled: rawScreenshotsEnabledSetting.get(), }; } diff --git a/app/native/g2mirror-client.ts b/app/native/g2mirror-client.ts new file mode 100644 index 00000000..28cb0608 --- /dev/null +++ b/app/native/g2mirror-client.ts @@ -0,0 +1,320 @@ +import { toUint8Array } from "../util/array-util"; + +declare const com: any; +declare const android: any; + +/** + * Client for the g2mirror terminal-mirroring protocol (see + * ../experiments/g2mirror/PROTOCOL.md). One instance per Terminal app + * session; talks JSON over a websocket to g2mirror-server, which relays to + * one wrapped CLI app at a time. + */ + +const PROTOCOL_VERSION = 1; +const SESSION_LIST_REFRESH_MS = 3_000; + +export type G2MirrorSession = { + socket: string; + pid: number; + cwdHint: string; +}; + +export type G2MirrorPhase = + | "idle" + | "connecting" + | "connected" // handshake accepted; can list/attach + | "attached" // relaying to one session + | "failed"; + +export type G2MirrorState = { + phase: G2MirrorPhase; + status: string; + sessions: G2MirrorSession[]; + attachedCommand: string; +}; + +export type G2MirrorClientOptions = { + host: string; + port: number; + authToken: string; + deviceName: string; + cols: number; + rows: number; +}; + +type TerminalDataKind = "snapshot" | "output"; + +export class G2MirrorClient { + private ws: any = null; + private listenerProxy: any = null; + private phase: G2MirrorPhase = "idle"; + private status = "Not connected."; + private sessions: G2MirrorSession[] = []; + private attachedCommand = ""; + private listRefreshTimer: ReturnType | null = null; + private stopped = false; + private readonly stateListeners = new Set<(state: G2MirrorState) => void>(); + private readonly terminalDataListeners = new Set<(data: Uint8Array, kind: TerminalDataKind) => void>(); + private readonly sessionAttachedListeners = new Set<(command: string) => void>(); + private readonly sessionDetachedListeners = new Set<(reason: string) => void>(); + + constructor(private readonly options: G2MirrorClientOptions) {} + + state(): G2MirrorState { + return { + phase: this.phase, + status: this.status, + sessions: this.sessions.slice(), + attachedCommand: this.attachedCommand, + }; + } + + onStateChange(listener: (state: G2MirrorState) => void): () => void { + this.stateListeners.add(listener); + return () => this.stateListeners.delete(listener); + } + + /** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */ + onTerminalData(listener: (data: Uint8Array, kind: TerminalDataKind) => void): () => void { + this.terminalDataListeners.add(listener); + return () => this.terminalDataListeners.delete(listener); + } + + onSessionAttached(listener: (command: string) => void): () => void { + this.sessionAttachedListeners.add(listener); + return () => this.sessionAttachedListeners.delete(listener); + } + + onSessionDetached(listener: (reason: string) => void): () => void { + this.sessionDetachedListeners.add(listener); + return () => this.sessionDetachedListeners.delete(listener); + } + + start(): void { + if (this.ws) return; + this.stopped = false; + const url = `ws://${this.options.host}:${this.options.port}`; + this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`); + this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + onOpen: () => { + if (this.stopped) return; + this.setState("connecting", "Authenticating..."); + this.send({ + type: "init", + version: PROTOCOL_VERSION, + auth_token: this.options.authToken, + device: this.options.deviceName, + width: this.options.cols, + height: this.options.rows, + }); + }, + onTextMessage: (message: string) => { + if (this.stopped) return; + this.handleMessage(String(message)); + }, + onClosed: (code: number, reason: string) => { + if (this.stopped) return; + this.handleConnectionLost(`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""}).`); + }, + onFailure: (message: string) => { + if (this.stopped) return; + this.handleConnectionLost(`Connection failed: ${shortenError(String(message))}`); + }, + }); + try { + this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy); + } catch (error) { + this.ws = null; + this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`); + } + } + + stop(): void { + this.stopped = true; + this.clearListRefreshTimer(); + if (this.ws) { + // Best effort: let the wrapped app resize back before we go away. + if (this.phase === "attached") { + this.send({ type: "unview" }); + this.send({ type: "disconnect" }); + } + try { + this.ws.close(1000, "bye"); + } catch { + // ignore + } + this.ws = null; + } + this.listenerProxy = null; + this.setState("idle", "Not connected."); + } + + listSessions(): void { + if (this.phase === "connected" || this.phase === "attached") { + this.send({ type: "list" }); + } + } + + connectSession(socket: string): void { + if (this.phase !== "connected") return; + this.setState(this.phase, "Attaching to session..."); + this.send({ type: "connect", socket }); + } + + disconnectSession(): void { + if (this.phase !== "attached") return; + this.send({ type: "unview" }); + this.send({ type: "disconnect" }); + } + + view(): void { + if (this.phase !== "attached") return; + this.send({ type: "view" }); + } + + unview(): void { + if (this.phase !== "attached") return; + this.send({ type: "unview" }); + } + + private handleMessage(text: string): void { + let message: any; + try { + message = JSON.parse(text); + } catch { + return; + } + if (!message || typeof message.type !== "string") return; + + switch (message.type) { + case "init": + this.setState("connected", "Connected."); + this.listSessions(); + this.ensureListRefreshTimer(); + return; + case "error": { + const errorText = String(message.message ?? "unknown error"); + if (this.phase === "connecting") { + // Handshake rejection; the server closes the socket after this. + this.setState("failed", `Rejected: ${errorText}`); + } else { + this.setState(this.phase, `Server error: ${errorText}`); + } + return; + } + case "sessions": { + const raw = Array.isArray(message.sessions) ? message.sessions : []; + this.sessions = raw + .map((item: any): G2MirrorSession => ({ + socket: String(item?.socket ?? ""), + pid: Number(item?.pid) || 0, + cwdHint: String(item?.cwd_hint ?? ""), + })) + .filter((session: G2MirrorSession) => session.socket.length > 0); + this.emitState(); + return; + } + case "connect": { + // Session accepted us (relayed from the wrapper). + this.clearListRefreshTimer(); + this.attachedCommand = String(message.command ?? ""); + this.setState("attached", `Attached: ${this.attachedCommand || "session"}`); + for (const listener of Array.from(this.sessionAttachedListeners)) { + listener(this.attachedCommand); + } + return; + } + case "snapshot": + case "output": { + const data = decodeBase64(String(message.data ?? "")); + for (const listener of Array.from(this.terminalDataListeners)) { + listener(data, message.type as TerminalDataKind); + } + return; + } + case "exit": { + const status = message.status === null || message.status === undefined ? "signal" : String(message.status); + this.setState(this.phase, `Session exited (status ${status}).`); + return; + } + case "disconnected": { + const reason = String(message.reason ?? "unknown"); + this.attachedCommand = ""; + if (this.phase === "attached") { + this.setState("connected", `Detached (${reason}).`); + this.ensureListRefreshTimer(); + this.listSessions(); + for (const listener of Array.from(this.sessionDetachedListeners)) { + listener(reason); + } + } + return; + } + default: + // Unknown message types are expected; ignore for forward compatibility. + return; + } + } + + private handleConnectionLost(statusText: string): void { + this.clearListRefreshTimer(); + this.ws = null; + this.listenerProxy = null; + const wasAttached = this.phase === "attached"; + this.attachedCommand = ""; + this.setState("failed", statusText); + if (wasAttached) { + for (const listener of Array.from(this.sessionDetachedListeners)) { + listener("connection lost"); + } + } + } + + private ensureListRefreshTimer(): void { + if (this.listRefreshTimer) return; + this.listRefreshTimer = setInterval(() => { + if (this.phase === "connected") { + this.listSessions(); + } + }, SESSION_LIST_REFRESH_MS); + } + + private clearListRefreshTimer(): void { + if (this.listRefreshTimer) { + clearInterval(this.listRefreshTimer); + this.listRefreshTimer = null; + } + } + + private send(message: object): void { + if (!this.ws) return; + try { + this.ws.sendText(JSON.stringify(message)); + } catch (error) { + console.warn("g2mirror send failed", error); + } + } + + private setState(phase: G2MirrorPhase, status: string): void { + this.phase = phase; + this.status = status; + this.emitState(); + } + + private emitState(): void { + const state = this.state(); + for (const listener of Array.from(this.stateListeners)) { + listener(state); + } + } +} + +function decodeBase64(data: string): Uint8Array { + if (!data) return new Uint8Array(0); + return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT)); +} + +function shortenError(message: string): string { + const collapsed = message.replace(/\s+/g, " ").trim(); + return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`; +} diff --git a/app/native/raw-screenshot.ts b/app/native/raw-screenshot.ts new file mode 100644 index 00000000..39e0ebc4 --- /dev/null +++ b/app/native/raw-screenshot.ts @@ -0,0 +1,24 @@ +import { Utils } from "@nativescript/core"; + +import { type GrayImage } from "../graphics/image"; + +declare const com: any; + +/** + * Save an image as the raw headerless 576x288 4bpp frame buffer (the exact + * bytes the wire compressor sees) for compression experiments. Returns the + * absolute path of the written file; retrieve with + * adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/ + */ +export function saveRawScreenshot(image: GrayImage): string { + const context = Utils.android.getApplicationContext(); + if (!context) throw new Error("Android application context unavailable"); + return String( + com.faceclaw.app.ScreenshotUtil.saveRawScreenshot( + context, + image.pixels.buffer, + image.width, + image.height, + ), + ); +} diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml index 72459a36..99311679 100644 --- a/app/phone-ui/main-page.xml +++ b/app/phone-ui/main-page.xml @@ -25,6 +25,7 @@