forked from Blendman974/faceclaw
[verified] merge upstream 25cb255b into all-features
This commit is contained in:
commit
ffcdae660d
169 files changed
+16201
-597
No files matched your search
@@ -55,8 +55,10 @@ export type GlanceboardProvider = {
|
||||
isEnabled: () => boolean;
|
||||
/** Whether a single tap shows the board (for tapTimeoutMs). */
|
||||
showOnTap: () => boolean;
|
||||
/** How long a tap or head-tilt keeps the board up. */
|
||||
/** How long a tap keeps the board up (and a head-tilt, when its head-down is unlikely to come). */
|
||||
tapTimeoutMs: () => number;
|
||||
/** How long a head-tilt showing may last without a head-down (null = no limit). */
|
||||
tiltTimeoutMs: () => number | null;
|
||||
/** Whether a long-press holds the board up until released. */
|
||||
showOnLongPress: () => boolean;
|
||||
/** Whether the head-tilt wake goes to the board rather than the regular UI. */
|
||||
@@ -187,6 +189,8 @@ export async function launchWorkerAppWindow(
|
||||
matchExistingBy?: "appId" | "windowId";
|
||||
/** Deliver watch swipes raw instead of the fallback (see WorkerWindowSpec). */
|
||||
acceptsDirectional?: boolean;
|
||||
/** Deliver hardware-keyboard keys as keys (see WorkerWindowSpec). */
|
||||
acceptsKeys?: boolean;
|
||||
},
|
||||
): Promise<void> {
|
||||
const existing = shell
|
||||
@@ -207,6 +211,7 @@ export async function launchWorkerAppWindow(
|
||||
icon: options.icon,
|
||||
focus: true,
|
||||
acceptsDirectional: options.acceptsDirectional,
|
||||
acceptsKeys: options.acceptsKeys,
|
||||
});
|
||||
ctx.appendLog(`launched ${options.windowId}`);
|
||||
}
|
||||
@@ -312,6 +312,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { hasCalendarPermission } from "../../native/calendar-permissions";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import type { Viewport } from "../../ui/viewport";
|
||||
import { listingDayMs, spanLabel, spansDays } from "./event-days";
|
||||
|
||||
/** How far a row's text sits inside its selection box; the title lines up with it. */
|
||||
const ROW_TEXT_INSET = 10;
|
||||
@@ -143,14 +144,20 @@ export class CalendarLayer implements Layer {
|
||||
function buildEventRows(font: UiFont, events: CalendarEvent[]): EventRow[] {
|
||||
const rows: EventRow[] = [];
|
||||
let previousDayKey = "";
|
||||
const nowMs = Date.now();
|
||||
for (const event of events) {
|
||||
const dayKey = dayKeyOf(event.startMs);
|
||||
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(event.startMs);
|
||||
// An event still going since an earlier day is listed under today, not its first day.
|
||||
const listedMs = listingDayMs(event, nowMs);
|
||||
const dayKey = dayKeyOf(listedMs);
|
||||
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(listedMs);
|
||||
previousDayKey = dayKey;
|
||||
|
||||
const lines: string[] = [];
|
||||
const timeLabel = event.allDay ? "All day" : formatEventTime(event.startMs);
|
||||
lines.push(`${timeLabel} ${event.title || "(untitled)"}`);
|
||||
if (spansDays(event)) {
|
||||
lines.push(spanLabel(event));
|
||||
}
|
||||
if (event.location) {
|
||||
lines.push(event.location);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Which day an event is listed under, and how a multi-day span reads. Pure (no
|
||||
* native or UI imports) so it can be unit-tested.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
export type DayEvent = { startMs: number; endMs: number; allDay: boolean };
|
||||
|
||||
function midnight(timestampMs: number): number {
|
||||
const date = new Date(timestampMs);
|
||||
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
||||
}
|
||||
|
||||
/** The last moment the event covers: all-day ends are exclusive (the midnight after its last day). */
|
||||
function lastMomentMs(event: DayEvent): number {
|
||||
return event.allDay ? Math.max(event.startMs, event.endMs - 1) : Math.max(event.startMs, event.endMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* The day to list an event under: its start, or today for one that began on an
|
||||
* earlier day and is still going. Without this, the third day of a three-day
|
||||
* event shows under its first day and never under today.
|
||||
*/
|
||||
export function listingDayMs(event: DayEvent, nowMs: number): number {
|
||||
return Math.max(midnight(event.startMs), Math.min(midnight(nowMs), midnight(lastMomentMs(event))));
|
||||
}
|
||||
|
||||
/** Whether the event covers more than one calendar day. */
|
||||
export function spansDays(event: DayEvent): boolean {
|
||||
return midnight(lastMomentMs(event)) > midnight(event.startMs);
|
||||
}
|
||||
|
||||
/** "Mon 16 – Wed 18 Nov", or with both months when they differ ("Mon 30 Nov – Tue 1 Dec"). */
|
||||
export function spanLabel(event: DayEvent): string {
|
||||
const start = new Date(event.startMs);
|
||||
const last = new Date(lastMomentMs(event));
|
||||
const day = (date: Date) => `${WEEKDAYS[date.getDay()]} ${date.getDate()}`;
|
||||
return start.getMonth() === last.getMonth() && start.getFullYear() === last.getFullYear()
|
||||
? `${day(start)} – ${day(last)} ${MONTHS[last.getMonth()]}`
|
||||
: `${day(start)} ${MONTHS[start.getMonth()]} – ${day(last)} ${MONTHS[last.getMonth()]}`;
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import { addImuListener, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
|
||||
import { addImuListener, imuPitchDegrees, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
@@ -23,7 +23,8 @@ const BAR_RIGHT_INSET = 4;
|
||||
/**
|
||||
* Accelerometer demo: turns the IMU stream on while this page is visible (the
|
||||
* Developer window is foregrounded and the screen is on), off otherwise, and
|
||||
* shows the latest x/y/z reading with a simple per-axis bar.
|
||||
* shows the latest x/y/z reading (Faceclaw's frame, imu-types.ts) with a
|
||||
* simple per-axis bar, and the head's pitch.
|
||||
*
|
||||
* The IMU stream is experimental — the firmware may never populate it — so the
|
||||
* page also reports how many samples have arrived.
|
||||
@@ -92,7 +93,8 @@ export class AccelerometerDemoLayer implements Layer {
|
||||
const bodyX = padding.left + BODY_INDENT;
|
||||
|
||||
image.drawText(small, padding.left, top, "Accelerometer", 220);
|
||||
const status = this.enabled ? "IMU on" : "IMU off";
|
||||
const pitch = this.reading ? `pitch ${Math.round(imuPitchDegrees(this.reading))}° ` : "";
|
||||
const status = pitch + (this.enabled ? "IMU on" : "IMU off");
|
||||
image.drawText(small, right - small.measureText(status), top, status, 150);
|
||||
|
||||
if (this.reading) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
|
||||
import { AccelerometerDemoLayer } from "./accelerometer-demo";
|
||||
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
|
||||
import { LightSensorDemoLayer } from "./light-sensor-demo";
|
||||
import { MouseTestLayer } from "./mouse-test";
|
||||
import { MicrophonesMonitorLayer } from "./microphones-monitor";
|
||||
import { ResourceUsageLayer } from "./resource-usage";
|
||||
import { RingStatusLayer } from "./ring-status";
|
||||
@@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
label: "Microphones",
|
||||
onSelect: openMicrophones,
|
||||
},
|
||||
{
|
||||
// The touchpad or mouse of a keyboard paired through the glasses.
|
||||
label: "Mouse cursor",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "BLE bandwidth",
|
||||
onSelect: (ctx) => {
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import type { HidPointerSample } from "../../g2/keyboard/hid";
|
||||
import { keyboardService } from "../../g2/keyboard/service";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
|
||||
// How often to reconcile holding the pointer with window visibility.
|
||||
const RECONCILE_INTERVAL_MS = 400;
|
||||
/** A touchpad reports a hundred times a second; the page repaints at most this often. */
|
||||
const RENDER_INTERVAL_MS = 33;
|
||||
/** Cursor pixels per count the touchpad reports. */
|
||||
const POINTER_GAIN = 1;
|
||||
/** Left-click marks kept on the page. */
|
||||
const MAX_MARKS = 12;
|
||||
|
||||
/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */
|
||||
const CURSOR = [
|
||||
"#",
|
||||
"##",
|
||||
"#.#",
|
||||
"#..#",
|
||||
"#...#",
|
||||
"#....#",
|
||||
"#.....#",
|
||||
"#......#",
|
||||
"#.......#",
|
||||
"#........#",
|
||||
"#.....#####",
|
||||
"#..#..#",
|
||||
"#.# #..#",
|
||||
"## #..#",
|
||||
"# #..#",
|
||||
" #..#",
|
||||
" ##",
|
||||
];
|
||||
|
||||
/**
|
||||
* Mouse cursor test: moves an arrow with the touchpad or mouse of the
|
||||
* keyboard paired through the glasses (app/g2/keyboard/), and shows its
|
||||
* buttons and wheel. The main UI has no cursor; this is where one is tried
|
||||
* out.
|
||||
*
|
||||
* While the page is visible it takes the pointer, so the buttons and wheel
|
||||
* show here instead of acting as select, back and scroll. A left click
|
||||
* leaves a mark where the arrow is and a right click clears them. Leave
|
||||
* with Esc, or a double tap on the ring or a temple.
|
||||
*/
|
||||
export class MouseTestLayer implements Layer {
|
||||
private x = -1;
|
||||
private y = -1;
|
||||
private size = { width: 0, height: 0 };
|
||||
private buttons = 0;
|
||||
private wheel = 0;
|
||||
private marks: Array<{ x: number; y: number }> = [];
|
||||
private samples = 0;
|
||||
private rate = 0;
|
||||
private rateWindowStart = Date.now();
|
||||
private removed = false;
|
||||
private release: (() => void) | null = null;
|
||||
private reconcileTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private renderTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly windowId: string,
|
||||
private readonly requestRender: () => void,
|
||||
) {
|
||||
this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS);
|
||||
this.reconcile();
|
||||
}
|
||||
|
||||
/** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */
|
||||
private reconcile(): void {
|
||||
if (this.removed) return;
|
||||
const visible = shell.isWindowVisible(this.windowId);
|
||||
if (visible && !this.release) {
|
||||
this.release = keyboardService.takePointer((sample) => this.onSample(sample));
|
||||
} else if (!visible && this.release) {
|
||||
this.release();
|
||||
this.release = null;
|
||||
this.buttons = 0;
|
||||
}
|
||||
// The rate reads zero once the pointer has been still for a second.
|
||||
const now = Date.now();
|
||||
if (now - this.rateWindowStart >= 1000) {
|
||||
const rate = this.samples;
|
||||
this.samples = 0;
|
||||
this.rateWindowStart = now;
|
||||
if (rate !== this.rate) {
|
||||
this.rate = rate;
|
||||
this.scheduleRender();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private onSample(sample: HidPointerSample): void {
|
||||
if (this.removed) return;
|
||||
this.samples++;
|
||||
const { width, height } = this.size;
|
||||
if (width > 0 && height > 0) {
|
||||
if (this.x < 0) {
|
||||
this.x = width / 2;
|
||||
this.y = height / 2;
|
||||
}
|
||||
this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1);
|
||||
this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1);
|
||||
}
|
||||
const pressed = sample.buttons & ~this.buttons;
|
||||
this.buttons = sample.buttons;
|
||||
this.wheel += sample.wheel;
|
||||
if (pressed & 1 && this.x >= 0) {
|
||||
this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) });
|
||||
if (this.marks.length > MAX_MARKS) this.marks.shift();
|
||||
}
|
||||
if (pressed & 2) this.marks = [];
|
||||
this.scheduleRender();
|
||||
}
|
||||
|
||||
private scheduleRender(): void {
|
||||
if (this.renderTimer !== null || this.removed) return;
|
||||
this.renderTimer = setTimeout(() => {
|
||||
this.renderTimer = null;
|
||||
if (!this.removed) this.requestRender();
|
||||
}, RENDER_INTERVAL_MS);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer);
|
||||
this.reconcileTimer = null;
|
||||
if (this.renderTimer !== null) clearTimeout(this.renderTimer);
|
||||
this.renderTimer = null;
|
||||
this.release?.();
|
||||
this.release = null;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height, padding } = ctx.stack.getBaseSize();
|
||||
this.size = { width, height };
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const left = padding.left;
|
||||
const right = width - padding.right;
|
||||
const top = padding.top;
|
||||
const line = font.lineHeight + 4;
|
||||
|
||||
image.drawText(font, left, top, "Mouse cursor", 220);
|
||||
const snapshot = keyboardService.snapshot();
|
||||
const problem = snapshot.state !== "connected"
|
||||
? "No keyboard is connected."
|
||||
: !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null;
|
||||
if (problem) {
|
||||
image.drawText(font, left, top + 2 * line, problem, 190);
|
||||
image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120);
|
||||
return image;
|
||||
}
|
||||
|
||||
const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`;
|
||||
image.drawText(font, right - font.measureText(position), top, position, 150);
|
||||
// Buttons: a box each, filled while held.
|
||||
let x = left;
|
||||
const boxY = top + line;
|
||||
for (const [index, label] of ["L", "M", "R"].entries()) {
|
||||
// HID numbers them left, right, middle; they sit left, middle, right.
|
||||
const held = !!(this.buttons & [1, 4, 2][index]!);
|
||||
const boxW = font.measureText(label) + 10;
|
||||
if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200);
|
||||
else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90);
|
||||
image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170);
|
||||
x += boxW + 4;
|
||||
}
|
||||
image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150);
|
||||
const rate = `${this.rate}/s`;
|
||||
image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110);
|
||||
const hint = "Left click marks, right click clears. Esc or double tap: back";
|
||||
image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90);
|
||||
|
||||
for (const mark of this.marks) {
|
||||
image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140);
|
||||
image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140);
|
||||
}
|
||||
if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y));
|
||||
return image;
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function drawCursor(image: GrayImage, x: number, y: number): void {
|
||||
for (const [row, pixels] of CURSOR.entries()) {
|
||||
for (let col = 0; col < pixels.length; col++) {
|
||||
const pixel = pixels[col];
|
||||
if (pixel === " ") continue;
|
||||
const px = x + col;
|
||||
const py = y + row;
|
||||
if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue;
|
||||
image.setPixel(px, py, pixel === "#" ? 255 : 70);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { isMicRecordingFile } from "../../native/mic-recording";
|
||||
import { isPlayableVideoFile } from "../../native/video-player";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
openFontWindow: (title: string, path: string) => void;
|
||||
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||
openVideoWindow: (title: string, path: string) => void;
|
||||
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
|
||||
openRecordingInTranscribe: (title: string, path: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) =>
|
||||
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isMicRecordingFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
label: "Open in Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openRecordingInTranscribe(entry.name, entry.path);
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isPlayableVideoFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
FILES_WINDOW_ID,
|
||||
} from "./files-app";
|
||||
import { createVideoPlayerWindow } from "./video-window";
|
||||
import { openRecordingInTranscribe } from "../transcribe";
|
||||
|
||||
// Document windows are closeable and non-singleton; the serial keeps their
|
||||
// windowIds unique.
|
||||
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
|
||||
},
|
||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||
openRecordingInTranscribe: (title, path) => {
|
||||
openRecordingInTranscribe(ctx, path, title);
|
||||
},
|
||||
}),
|
||||
),
|
||||
// A document arriving via Android's Share intent opens as its own window.
|
||||
|
||||
@@ -305,6 +305,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
break;
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -236,6 +236,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
case "navigation-sensors":
|
||||
case "screen":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from
|
||||
import { uiFontOverridePickerMenuItem } from "../../ui/font-picker";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, menuHeadingItem, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { liveHeadPitchFooter } from "../../g2/head-angle";
|
||||
import { headUpControlsAvailable } from "../../native/glasses-controls";
|
||||
import { Menu } from "../../ui/menu-core";
|
||||
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
|
||||
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
@@ -20,7 +22,9 @@ import {
|
||||
glanceLayout,
|
||||
glanceLayoutSetting,
|
||||
glanceShowLinesSetting,
|
||||
glanceAngleSetting,
|
||||
glanceShowOnHeadTiltSetting,
|
||||
glanceTiltTimeoutSetting,
|
||||
glanceShowOnLongPressSetting,
|
||||
glanceSlotChoiceLabel,
|
||||
glanceSlotSettings,
|
||||
@@ -319,25 +323,45 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings: layout, sleep gestures, duration, slot lines, widget font, and
|
||||
* stereo depth, then a row per widget on the board for its own settings.
|
||||
* Settings, in four sections: Layout (the slot grid), Trigger (the sleep
|
||||
* gestures that show the board, and for how long), Display (slot lines,
|
||||
* widget font, stereo depth) and Cards (a row per widget on the board, for
|
||||
* its own settings; left out while the board is empty).
|
||||
*/
|
||||
function openGlanceSettings(ctx: LayerContext): void {
|
||||
const items = (): MenuItem[] => [
|
||||
// The layout decides which slots (and so which widgets) are on the board.
|
||||
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
toggleSettingMenuItem(glanceShowLinesSetting),
|
||||
uiFontOverridePickerMenuItem(glanceFont, {
|
||||
title: "Glanceboard font",
|
||||
description:
|
||||
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
|
||||
}),
|
||||
enumSettingMenuItem(glanceDepthSetting),
|
||||
...widgetSettingsMenuItems(),
|
||||
];
|
||||
const items = (): MenuItem[] => {
|
||||
const cards = widgetSettingsMenuItems();
|
||||
return [
|
||||
menuHeadingItem("Layout"),
|
||||
// The layout decides which slots (and so which widgets) are on the board.
|
||||
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
|
||||
|
||||
menuHeadingItem("Trigger"),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
enumSettingMenuItem(glanceTiltTimeoutSetting),
|
||||
// Only Android can change the glasses' head-up setting (glasses-controls.ts).
|
||||
// The controller's settings listener sends a new angle to the glasses; the
|
||||
// picker shows the head's current angle to choose by.
|
||||
...(headUpControlsAvailable()
|
||||
? [enumSettingMenuItem(glanceAngleSetting, {
|
||||
pickerFooter: liveHeadPitchFooter(() => shell.isWindowVisible(GLANCEBOARD_WINDOW_ID)),
|
||||
})]
|
||||
: []),
|
||||
|
||||
menuHeadingItem("Display"),
|
||||
toggleSettingMenuItem(glanceShowLinesSetting),
|
||||
uiFontOverridePickerMenuItem(glanceFont, {
|
||||
title: "Glanceboard font",
|
||||
description:
|
||||
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
|
||||
}),
|
||||
enumSettingMenuItem(glanceDepthSetting),
|
||||
|
||||
...(cards.length ? [menuHeadingItem("Cards"), ...cards] : []),
|
||||
];
|
||||
};
|
||||
const layer = new MenuLayer("Glanceboard settings", items(), PAGE_MENU_LAYOUT);
|
||||
ctx.stack.push(layer);
|
||||
}
|
||||
|
||||
@@ -49,8 +49,9 @@ export function glanceLayout(): GlanceLayout {
|
||||
|
||||
/**
|
||||
* Whether sleep-time gestures show the board at all. Off restores the plain
|
||||
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
|
||||
* wakes the regular UI), double-tap wakes.
|
||||
* behaviour: a tap or hold while asleep does nothing, a head-tilt wakes the
|
||||
* regular UI, double-tap wakes; and the head-up angle goes back to the glasses'
|
||||
* own.
|
||||
*/
|
||||
export const glanceboardEnabledSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-enabled",
|
||||
@@ -85,9 +86,10 @@ function formatTapDuration(value: GlanceTapDuration): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* How long a tap (or head-tilt) keeps the board up; each further tap restarts
|
||||
* it. "off" means a single tap does not show the board at all; a head-tilt
|
||||
* then uses the default duration.
|
||||
* How long a tap keeps the board up; each further tap restarts it. "off"
|
||||
* means a single tap does not show the board at all. A head-tilt showing
|
||||
* lasts until the head comes back down, falling back to this duration (the
|
||||
* default one when "off") where the head-down is unlikely to arrive.
|
||||
*/
|
||||
export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>({
|
||||
id: "glanceboard-tap-duration",
|
||||
@@ -97,7 +99,7 @@ export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>
|
||||
values: TAP_DURATION_VALUES,
|
||||
formatValue: formatTapDuration,
|
||||
description:
|
||||
"How long a single tap (or a head-tilt) keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
|
||||
"How long a single tap keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
|
||||
});
|
||||
|
||||
/** Whether a single tap while asleep shows the board. */
|
||||
@@ -119,15 +121,71 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
|
||||
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
|
||||
});
|
||||
|
||||
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
|
||||
/** The head-tilt wake shows the board (until the head comes back down) instead of the regular UI. */
|
||||
export const glanceShowOnHeadTiltSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-show-on-head-tilt",
|
||||
label: "Show on head tilt",
|
||||
storageKey: "glanceboard.showOnHeadTilt",
|
||||
defaultValue: true,
|
||||
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI.",
|
||||
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI. It stays up until you look back down.",
|
||||
});
|
||||
|
||||
export type GlanceTiltTimeout = "60s" | "30s" | "15s" | "10s" | "never";
|
||||
|
||||
/**
|
||||
* How long a head-tilt showing may last if the head-down never comes (the
|
||||
* board normally hides when the head comes back down). "never" removes the
|
||||
* limit; a double-tap still dismisses the board.
|
||||
*/
|
||||
export const glanceTiltTimeoutSetting = new ConfigSettingEnum<GlanceTiltTimeout>({
|
||||
id: "glanceboard-head-tilt-timeout",
|
||||
label: "Head tilt timeout",
|
||||
storageKey: "glanceboard.headTiltTimeout",
|
||||
defaultValue: "60s",
|
||||
values: ["60s", "30s", "15s", "10s", "never"],
|
||||
formatValue: (value) => (value === "never" ? "Never" : value.replace("s", " s")),
|
||||
description:
|
||||
"The longest a head-tilt keeps the Glanceboard up when looking back down is not detected. Never: it stays until you look down or double-tap. " +
|
||||
"When looking down is unlikely to register (a low Head tilt angle, or the display was suspended), the tap duration applies if shorter.",
|
||||
});
|
||||
|
||||
/** The Head tilt timeout in ms, or null for none. */
|
||||
export function glanceTiltTimeoutMs(): number | null {
|
||||
const value = glanceTiltTimeoutSetting.get();
|
||||
return value === "never" ? null : Number.parseInt(value, 10) * 1000;
|
||||
}
|
||||
|
||||
const GLANCE_ANGLE_VALUES = ["glasses", "5", "10", "15", "20", "25", "30", "35", "40", "45", "50", "55", "60"] as const;
|
||||
export type GlanceAngle = (typeof GLANCE_ANGLE_VALUES)[number];
|
||||
|
||||
/**
|
||||
* How far the head must tilt up for a head-tilt wake, as the glasses' own
|
||||
* pitch reading (degrees). "glasses" keeps the glasses' own setting (from the
|
||||
* Even app), writing back their original values if Faceclaw changed them. A
|
||||
* number is written to the glasses' head-up setting on every connection while
|
||||
* the Glanceboard is enabled (head-angle.ts). Android only.
|
||||
*/
|
||||
export const glanceAngleSetting = new ConfigSettingEnum<GlanceAngle>({
|
||||
id: "glanceboard-head-tilt-angle",
|
||||
label: "Head tilt angle",
|
||||
storageKey: "glanceboard.headTiltAngle",
|
||||
defaultValue: "glasses",
|
||||
values: GLANCE_ANGLE_VALUES,
|
||||
formatValue: (value) => (value === "glasses" ? "Glasses default" : `${value}°`),
|
||||
description:
|
||||
"How far you tilt your head up to wake the display; the list shows your head's angle now. Glasses default uses the glasses' own setting from the Even app. " +
|
||||
"The board hides when your head drops 20° below this angle; under 25°, looking straight ahead may not count, so it hides after the tap duration instead.",
|
||||
});
|
||||
|
||||
/**
|
||||
* The trigger Faceclaw should maintain, in degrees: the Head tilt angle while
|
||||
* the Glanceboard is enabled, else 0 (the glasses' own setting).
|
||||
*/
|
||||
export function glanceAngleTriggerDegrees(): number {
|
||||
const value = glanceAngleSetting.get();
|
||||
return value === "glasses" || !glanceboardEnabledSetting.get() ? 0 : Number(value);
|
||||
}
|
||||
|
||||
const DEPTH_VALUES = ["-64", "-48", "-32", "-16", "0", "16", "32", "48", "64"] as const;
|
||||
export type GlanceDepth = (typeof DEPTH_VALUES)[number];
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
glanceShowOnLongPressSetting,
|
||||
glanceShowOnTap,
|
||||
glanceTapTimeoutMs,
|
||||
glanceTiltTimeoutMs,
|
||||
} from "./glanceboard-settings";
|
||||
|
||||
const glanceboardApp: AppDefinition = {
|
||||
@@ -24,6 +25,7 @@ const glanceboardApp: AppDefinition = {
|
||||
isEnabled: () => glanceboardEnabledSetting.get(),
|
||||
showOnTap: glanceShowOnTap,
|
||||
tapTimeoutMs: glanceTapTimeoutMs,
|
||||
tiltTimeoutMs: glanceTiltTimeoutMs,
|
||||
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
|
||||
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
|
||||
depth: glanceDepth,
|
||||
|
||||
@@ -50,16 +50,26 @@ export const musicShowAlbumArtSetting = new ConfigSettingBoolean({
|
||||
"Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.",
|
||||
});
|
||||
|
||||
export const musicShowIdleNoteSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-music-show-idle-note",
|
||||
label: "Show note when idle",
|
||||
storageKey: "glanceboard.music.showIdleNote",
|
||||
defaultValue: true,
|
||||
description: "Show a dim music note on the card while nothing is playing. When off, the card is left blank.",
|
||||
});
|
||||
|
||||
/** The music card's submenu in Glanceboard > Settings. */
|
||||
export function musicSettingsMenuItems(): MenuItem[] {
|
||||
return [toggleSettingMenuItem(musicShowAlbumArtSetting)];
|
||||
return [toggleSettingMenuItem(musicShowAlbumArtSetting), toggleSettingMenuItem(musicShowIdleNoteSetting)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Now playing: album art (unless turned off), title/artist/album and a
|
||||
* progress bar for the active media session (Apple Music on iOS), the Music
|
||||
* app's header at card size. The media bridge runs for the whole session; the
|
||||
* widget subscribes and ticks once a second only while something is playing.
|
||||
* app's header at card size. With nothing playing the card holds a dim note,
|
||||
* or nothing with that turned off. The media bridge runs for the whole
|
||||
* session; the widget subscribes and ticks once a second only while
|
||||
* something is playing.
|
||||
*/
|
||||
export class MusicWidget implements GlanceWidget {
|
||||
private media: MediaControllerState = mediaControllerBridge.snapshot();
|
||||
@@ -108,8 +118,8 @@ export class MusicWidget implements GlanceWidget {
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
|
||||
return;
|
||||
}
|
||||
// Nothing playing: a dim note, centred, and nothing else.
|
||||
const note = idleNoteImage();
|
||||
// Nothing playing: a dim note, centred, and nothing else; or a blank card.
|
||||
const note = musicShowIdleNoteSetting.get() ? idleNoteImage() : null;
|
||||
if (note) image.drawImage(note, ((image.width - note.width) / 2) | 0, ((image.height - note.height) / 2) | 0);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration a capture of the temples' own microphones arms them
|
||||
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
|
||||
* of each. The front end and direction detection follow Microphone setup
|
||||
* (the Microphones app), except that both of a temple's microphones only
|
||||
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
|
||||
* what the firmware streams and transcription takes.
|
||||
*/
|
||||
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
|
||||
const base = loadMicConfig();
|
||||
return {
|
||||
...base,
|
||||
source: mics === 4 ? "codec" : base.source,
|
||||
channelMask: mics === 4 ? 0b11 : 0b01,
|
||||
codec: "raw",
|
||||
format: "pcm16",
|
||||
sampleRateHz: 16_000,
|
||||
armHardware: true,
|
||||
};
|
||||
}
|
||||
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
|
||||
@@ -272,6 +272,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -370,6 +370,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (editing) {
|
||||
// Voice input as an alternative to the phone keyboard.
|
||||
|
||||
@@ -433,6 +433,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -227,6 +227,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (editingSetting) {
|
||||
// Voice input as an alternative to the phone keyboard.
|
||||
|
||||
@@ -424,6 +424,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (window && window.windowId === message.windowId) handleTextInput(message.text.trim());
|
||||
break;
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
|
||||
/**
|
||||
* Terminal-app connection registry: the g2mirror hosts the app knows about,
|
||||
* stored as JSON under one settings key so every isolate sees the same list
|
||||
* and change notifications ride the ordinary settings-store channel.
|
||||
* Terminal-app connection registry: the hosts the app knows about, stored as
|
||||
* JSON under one settings key so every isolate sees the same list and change
|
||||
* notifications ride the ordinary settings-store channel.
|
||||
*
|
||||
* A connection is entered as a g2mirror:// connection string (the same form
|
||||
* g2mirror-view accepts): g2mirror://<token>@<host>[:port] for plain
|
||||
* websockets (default port 8737), g2mirrors://... for TLS (default 443).
|
||||
* A connection is entered as a connection string:
|
||||
* - g2mirror://<token>@<host>[:port]: a g2mirror server reached directly
|
||||
* (the same form g2mirror-view accepts) over a plain websocket (default
|
||||
* port 8737); g2mirrors://... for TLS (default 443).
|
||||
* - ssh://<user>@<host>[:port]: tmux on an SSH host (default port 22).
|
||||
* - ssh://<user>@<host>[:port]?g2mirror=<token>[@<host>[:port]]: a g2mirror
|
||||
* server tunnelled over SSH; the address after the token is where the
|
||||
* server listens as seen from the SSH host (default 127.0.0.1:8737).
|
||||
* SSH connections pin the host's key (hostKeys, trusted on first connection
|
||||
* after the user checks its fingerprint); their passwords live in the
|
||||
* platform keystore (app/native/ssh), not here.
|
||||
*/
|
||||
|
||||
export const TERMINAL_CONNECTIONS_KEY = "terminal.connections";
|
||||
@@ -25,6 +33,8 @@ export type TerminalConnection = {
|
||||
serverName: string | null;
|
||||
/** Auto-connect while the Terminal app is open. Toggled by Connect/Disconnect. */
|
||||
enabled: boolean;
|
||||
/** SSH connections: trusted host keys (SSH wire format, base64). */
|
||||
hostKeys: string[];
|
||||
};
|
||||
|
||||
export type ParsedConnectionString = {
|
||||
@@ -54,8 +64,79 @@ export function parseConnectionString(url: string): ParsedConnectionString | nul
|
||||
return { secure, host, port, authToken };
|
||||
}
|
||||
|
||||
/** The host[:port] part of a connection string; "" if it doesn't parse. */
|
||||
export type ParsedSshConnection = {
|
||||
user: string;
|
||||
host: string;
|
||||
port: number;
|
||||
} & (
|
||||
| { mode: "tmux" }
|
||||
| {
|
||||
mode: "g2mirror";
|
||||
authToken: string;
|
||||
/** Where g2mirror-server listens, from the SSH host's point of view. */
|
||||
targetHost: string;
|
||||
targetPort: number;
|
||||
}
|
||||
);
|
||||
|
||||
function decodeComponent(text: string): string {
|
||||
try {
|
||||
return decodeURIComponent(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse an ssh://user@host[:port][?g2mirror=token[@host[:port]]] string; null if malformed. */
|
||||
export function parseSshConnectionString(url: string): ParsedSshConnection | null {
|
||||
const match = /^ssh:\/\/([^@\s/?]+)@(\[[^\]\s]+\]|[^:/@\s?]+)(?::(\d+))?\/?(?:\?(.*))?$/.exec(url.trim());
|
||||
if (!match) return null;
|
||||
const user = decodeComponent(match[1]!);
|
||||
const host = match[2]!.replace(/^\[(.*)\]$/, "$1");
|
||||
const port = match[3] ? parseInt(match[3], 10) : 22;
|
||||
if (!user || !(port >= 1 && port <= 65535)) return null;
|
||||
const params = new Map<string, string>();
|
||||
for (const part of (match[4] ?? "").split("&").filter(Boolean)) {
|
||||
const equals = part.indexOf("=");
|
||||
if (equals < 0) return null;
|
||||
params.set(part.slice(0, equals), decodeComponent(part.slice(equals + 1)));
|
||||
}
|
||||
for (const key of params.keys()) {
|
||||
if (key !== "g2mirror") return null;
|
||||
}
|
||||
const g2mirror = params.get("g2mirror");
|
||||
if (g2mirror === undefined) return { user, host, port, mode: "tmux" };
|
||||
// token[@host[:port]], split at the last "@" so the token may contain one.
|
||||
const at = g2mirror.lastIndexOf("@");
|
||||
const authToken = at < 0 ? g2mirror : g2mirror.slice(0, at);
|
||||
const target = at < 0 ? "" : g2mirror.slice(at + 1);
|
||||
const targetMatch = /^(\[[^\]\s]+\]|[^:\s]*)(?::(\d+))?$/.exec(target);
|
||||
if (!authToken || !targetMatch) return null;
|
||||
const targetHost = targetMatch[1] || "127.0.0.1";
|
||||
const targetPort = targetMatch[2] ? parseInt(targetMatch[2], 10) : 8737;
|
||||
if (!(targetPort >= 1 && targetPort <= 65535)) return null;
|
||||
return { user, host, port, mode: "g2mirror", authToken, targetHost, targetPort };
|
||||
}
|
||||
|
||||
export type ParsedAnyConnection =
|
||||
| ({ kind: "g2mirror" } & ParsedConnectionString)
|
||||
| ({ kind: "ssh" } & ParsedSshConnection);
|
||||
|
||||
/** Parse any supported connection string; null if it's none of them. */
|
||||
export function parseAnyConnectionString(url: string): ParsedAnyConnection | null {
|
||||
const direct = parseConnectionString(url);
|
||||
if (direct) return { kind: "g2mirror", ...direct };
|
||||
const ssh = parseSshConnectionString(url);
|
||||
return ssh ? { kind: "ssh", ...ssh } : null;
|
||||
}
|
||||
|
||||
/** The [user@]host[:port] part of a connection string (never a token); "" if it doesn't parse. */
|
||||
export function connectionStringHostLabel(url: string): string {
|
||||
const ssh = parseSshConnectionString(url);
|
||||
if (ssh) {
|
||||
const host = ssh.host.includes(":") ? `[${ssh.host}]` : ssh.host;
|
||||
return `${ssh.user}@${host}${ssh.port === 22 ? "" : `:${ssh.port}`}`;
|
||||
}
|
||||
const parsed = parseConnectionString(url);
|
||||
if (!parsed) return "";
|
||||
const defaultPort = parsed.secure ? 443 : 8737;
|
||||
@@ -79,6 +160,7 @@ export function loadConnections(): TerminalConnection[] {
|
||||
url: String(item?.url ?? ""),
|
||||
serverName: typeof item?.serverName === "string" && item.serverName ? item.serverName : null,
|
||||
enabled: item?.enabled !== false,
|
||||
hostKeys: Array.isArray(item?.hostKeys) ? item.hostKeys.filter((key: unknown) => typeof key === "string") : [],
|
||||
}))
|
||||
.filter((connection) => connection.id.length > 0 && connection.url.length > 0);
|
||||
} catch {
|
||||
|
||||
@@ -14,6 +14,8 @@ const terminalApp: AppDefinition = {
|
||||
// Session windows stay independent: launching reopens the hub even
|
||||
// while terminal sessions are open.
|
||||
matchExistingBy: "windowId",
|
||||
// The hub window can show a session too, so it takes keys like one.
|
||||
acceptsKeys: true,
|
||||
}),
|
||||
};
|
||||
|
||||
|
||||
@@ -16,11 +16,17 @@
|
||||
* Each view has its own websocket connection (the protocol allows one attached
|
||||
* session per connection) and its own xterm emulator.
|
||||
*
|
||||
* Connections are g2mirror://<token>@<host>[:port] strings (g2mirrors:// for
|
||||
* TLS), stored as JSON in the settings store (see connections.ts). Each
|
||||
* enabled connection gets a control websocket that backs the session list and
|
||||
* carries unsolicited bell/title notifications; control connections live as
|
||||
* long as the worker so bells keep flowing even if the hub window is closed.
|
||||
* Connections are connection strings stored as JSON in the settings store
|
||||
* (see connections.ts): g2mirror://<token>@<host>[:port] for a g2mirror
|
||||
* server (g2mirrors:// for TLS), ssh://<user>@<host> for tmux over SSH
|
||||
* (TmuxClient), or ssh://<user>@<host>?g2mirror=<token> for a g2mirror server
|
||||
* tunnelled over SSH. Each enabled connection gets a control client that
|
||||
* backs the session list and carries bell/title/activity notifications;
|
||||
* control connections live as long as the worker so bells keep flowing even
|
||||
* if the hub window is closed. SSH connections that need the user (an
|
||||
* unverified or changed host key, a password, an unauthorized key) record
|
||||
* the failure on the control (sshIssue) and offer a page that resolves it
|
||||
* instead of retrying.
|
||||
* The host's server_name (from the init reply) is cached per connection and
|
||||
* used as its display name, falling back to the connection string's
|
||||
* host[:port] before the first successful handshake.
|
||||
@@ -55,7 +61,24 @@ import {
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
import { encodeTerminalKey, terminalScrollbackKey } from "./terminal-keys";
|
||||
import { G2MirrorClient, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
|
||||
import type { TerminalClient } from "./terminal-client";
|
||||
import { TmuxClient, type OpenLineChannel } from "./tmux-client";
|
||||
import {
|
||||
copyToPhoneClipboard,
|
||||
describeSshFailure,
|
||||
getSshSecret,
|
||||
openSshChannel,
|
||||
removeSshSecret,
|
||||
setSshSecret,
|
||||
sshClientKey,
|
||||
type SshEndpoint,
|
||||
type SshFailure,
|
||||
type SshHostKey,
|
||||
} from "../../native/ssh";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import { onSettingsStoreChanged } from "../../native/settings-store";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import {
|
||||
@@ -64,12 +87,22 @@ import {
|
||||
terminalDisplayLayoutSetting,
|
||||
terminalLaunchPresetsSetting,
|
||||
terminalNewConnectionSetting,
|
||||
terminalSshPasswordSetting,
|
||||
terminalWakeOnBellSetting,
|
||||
textSettingMenuItem,
|
||||
toggleSettingMenuItem,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { terminalFontPickerMenuItem } from "../../ui/font-picker";
|
||||
import { connectionDisplayName, loadConnections, parseConnectionString, saveConnections, TERMINAL_CONNECTIONS_KEY, updateConnection, type TerminalConnection } from "./connections";
|
||||
import {
|
||||
connectionDisplayName,
|
||||
connectionStringHostLabel,
|
||||
loadConnections,
|
||||
parseAnyConnectionString,
|
||||
saveConnections,
|
||||
TERMINAL_CONNECTIONS_KEY,
|
||||
updateConnection,
|
||||
type TerminalConnection,
|
||||
} from "./connections";
|
||||
import { TerminalEmulator } from "./terminal-emulator";
|
||||
import { drawSubmenuIndicator, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
@@ -112,6 +145,10 @@ const RECONNECT_MAX_DELAY_MS = 60_000;
|
||||
const NEW_CONNECTION_DRAFT_KEY = "terminal.newConnectionDraft";
|
||||
// Storage key of terminalLaunchPresetsSetting, edited from the window menu.
|
||||
const LAUNCH_PRESETS_KEY = "terminal.launchPresets";
|
||||
// Storage key of terminalSshPasswordSetting (the password page's draft).
|
||||
const SSH_PASSWORD_DRAFT_KEY = "terminal.sshPasswordDraft";
|
||||
// Comment on this device's SSH public key in authorized_keys.
|
||||
const SSH_KEY_COMMENT = "faceclaw";
|
||||
|
||||
type BaseWindow = {
|
||||
windowId: string;
|
||||
@@ -148,10 +185,24 @@ type BaseWindow = {
|
||||
|
||||
/**
|
||||
* What the hub window is showing: the session list, the Manage Connections
|
||||
* section, or the Add-connection screen (type the g2mirror:// string on the
|
||||
* phone, click to save).
|
||||
* section, the Add-connection screen (type the connection string on the
|
||||
* phone, click to save), or a page of text with a click action (verify a
|
||||
* host key, enter a password, show this device's SSH key).
|
||||
*/
|
||||
type HubMode = "sessions" | "connections" | "add";
|
||||
type HubMode = "sessions" | "connections" | "add" | "page";
|
||||
|
||||
type HubPage = {
|
||||
title: string;
|
||||
/** Body text, wrapped to the window; re-read on every paint. */
|
||||
body: () => string;
|
||||
/** Bottom hint line. */
|
||||
hint: string;
|
||||
onClick?: () => void;
|
||||
/** Double-click. */
|
||||
onBack: () => void;
|
||||
/** Runs whenever the page goes away (back, click, window closed). */
|
||||
onLeave?: () => void;
|
||||
};
|
||||
|
||||
type HubWindow = BaseWindow & {
|
||||
kind: "hub";
|
||||
@@ -168,6 +219,8 @@ type HubWindow = BaseWindow & {
|
||||
sessionOrder: string[];
|
||||
/** Parse-failure message shown on the Add-connection screen. */
|
||||
addError: string;
|
||||
/** The page shown in mode "page". */
|
||||
page: HubPage | null;
|
||||
};
|
||||
|
||||
type ViewWindow = BaseWindow & {
|
||||
@@ -186,7 +239,7 @@ type ViewWindow = BaseWindow & {
|
||||
label: string;
|
||||
/** Sidebar-icon character (">3"); "" if every glyph was taken at open time. */
|
||||
glyph: string;
|
||||
client: G2MirrorClient;
|
||||
client: TerminalClient;
|
||||
emulator: TerminalEmulator;
|
||||
receivedData: boolean;
|
||||
attachRequested: boolean;
|
||||
@@ -220,8 +273,8 @@ type PendingView = {
|
||||
socket: string;
|
||||
label: string;
|
||||
glyph: string;
|
||||
/** Client options captured when the view was requested from a live control. */
|
||||
options: G2MirrorClientOptions;
|
||||
/** How to make the view's client, captured when it was requested from a live control. */
|
||||
options: ClientRecipe;
|
||||
};
|
||||
|
||||
const windows = new Map<string, TerminalWindow>();
|
||||
@@ -302,7 +355,13 @@ let screenOn = true;
|
||||
*/
|
||||
type ControlConnection = {
|
||||
config: TerminalConnection;
|
||||
client: G2MirrorClient | null;
|
||||
client: TerminalClient | null;
|
||||
/**
|
||||
* An SSH failure only the user can fix (host key to verify, password or
|
||||
* key needed). Shown on the host's heading with a row that opens the fix;
|
||||
* reconnects wait for it. Cleared on a successful connection.
|
||||
*/
|
||||
sshIssue: SshFailure | null;
|
||||
state: G2MirrorState | null;
|
||||
unsubscribers: Array<() => void>;
|
||||
reconnectTimer: ReturnType<typeof setTimeout> | null;
|
||||
@@ -435,6 +494,13 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "key": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
window.focused = message.focused;
|
||||
handleKey(window, message.event as KeyEvent);
|
||||
break;
|
||||
}
|
||||
case "render": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
@@ -504,7 +570,8 @@ onSettingsStoreChanged((key) => {
|
||||
reportIdle();
|
||||
return;
|
||||
case NEW_CONNECTION_DRAFT_KEY:
|
||||
// Live keystrokes from the phone editor; repaint the add screen.
|
||||
case SSH_PASSWORD_DRAFT_KEY:
|
||||
// Live keystrokes from the phone editor; repaint the add/password screen.
|
||||
renderHubWindows();
|
||||
return;
|
||||
case "terminal.autoReconnect":
|
||||
@@ -578,6 +645,7 @@ function openWindow(windowId: string, surfaceId: string, title: string, viewport
|
||||
list: null,
|
||||
sessionOrder: [],
|
||||
addError: "",
|
||||
page: null,
|
||||
});
|
||||
// The hub carries the app's assistant tools; declare them once it exists.
|
||||
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
|
||||
@@ -610,6 +678,10 @@ function closeWindow(windowId: string): void {
|
||||
// Window closed mid-add: shut the phone editor down.
|
||||
endAddConnection(window);
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "page") {
|
||||
// Likewise for a page's phone editor (the password draft).
|
||||
closeHubPage(window, "sessions");
|
||||
}
|
||||
// Likewise for a setting being edited from the window menu.
|
||||
window.menu?.close();
|
||||
windows.delete(windowId);
|
||||
@@ -627,17 +699,120 @@ function closeWindow(windowId: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** Client options for a connection string; null if the string doesn't parse. */
|
||||
function clientOptionsFor(connection: TerminalConnection): G2MirrorClientOptions | null {
|
||||
const parsed = parseConnectionString(connection.url);
|
||||
/**
|
||||
* How to make clients for a connection: "g2mirror" dials a g2mirror server
|
||||
* directly, "ssh-g2mirror" reaches one through an SSH tunnel, "tmux" runs
|
||||
* tmux control mode over SSH.
|
||||
*/
|
||||
type ClientRecipe = {
|
||||
kind: "g2mirror" | "ssh-g2mirror" | "tmux";
|
||||
/**
|
||||
* `role` "control" watches bells (tmux keeps the bell bookkeeping for that
|
||||
* client only). `onSshFailure` hears why an SSH channel failed.
|
||||
*/
|
||||
create(cols: number, rows: number, role: "control" | "view", onSshFailure?: (failure: SshFailure) => void): TerminalClient;
|
||||
};
|
||||
|
||||
/** Client recipe for a connection string; null if the string doesn't parse. */
|
||||
function clientOptionsFor(connection: TerminalConnection): ClientRecipe | null {
|
||||
const parsed = parseAnyConnectionString(connection.url);
|
||||
if (!parsed) return null;
|
||||
if (parsed.kind === "g2mirror") {
|
||||
const { secure, host, port, authToken } = parsed;
|
||||
return {
|
||||
kind: "g2mirror",
|
||||
create: (cols, rows) => new G2MirrorClient({ secure, host, port, authToken, deviceName: DEVICE_NAME, cols, rows }),
|
||||
};
|
||||
}
|
||||
const connectionId = connection.id;
|
||||
const label = connectionStringHostLabel(connection.url);
|
||||
// Read at every channel open, so a newly trusted key or saved password
|
||||
// applies to the next (re)connect, views included.
|
||||
const endpoint = (): SshEndpoint => ({
|
||||
host: parsed.host,
|
||||
port: parsed.port,
|
||||
user: parsed.user,
|
||||
hostKeys: loadConnections().find((candidate) => candidate.id === connectionId)?.hostKeys ?? [],
|
||||
password: getSshSecret(sshPasswordSecretId(connectionId)),
|
||||
useKey: true,
|
||||
});
|
||||
if (parsed.mode === "g2mirror") {
|
||||
const { authToken, targetHost, targetPort } = parsed;
|
||||
const urlHost = targetHost.includes(":") && !targetHost.startsWith("[") ? `[${targetHost}]` : targetHost;
|
||||
const url = `ws://${urlHost}:${targetPort}/`;
|
||||
return {
|
||||
kind: "ssh-g2mirror",
|
||||
create: (cols, rows, _role, onSshFailure) =>
|
||||
new G2MirrorClient({
|
||||
secure: false,
|
||||
host: targetHost,
|
||||
port: targetPort,
|
||||
authToken,
|
||||
deviceName: DEVICE_NAME,
|
||||
cols,
|
||||
rows,
|
||||
transport: {
|
||||
label: `${label} over SSH`,
|
||||
open: (listener) => {
|
||||
const channel = openSshChannel(endpoint(), { type: "websocket", url }, {
|
||||
onOpen: () => listener.onOpen(),
|
||||
onText: (text) => listener.onTextMessage(text),
|
||||
onClosed: (info) => listener.onClosed(info.code ?? 1000, info.reason ?? ""),
|
||||
onFailure: (failure) => {
|
||||
onSshFailure?.(failure);
|
||||
listener.onFailure(describeSshFailure(failure));
|
||||
},
|
||||
});
|
||||
return { sendText: (text) => channel.send(text), sendBinary: () => false, close: () => channel.close() };
|
||||
},
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
...parsed,
|
||||
deviceName: DEVICE_NAME,
|
||||
...viewGrid(),
|
||||
kind: "tmux",
|
||||
create: (cols, rows, role, onSshFailure) => {
|
||||
const openChannel: OpenLineChannel = (command, listener) =>
|
||||
openSshChannel(endpoint(), { type: "exec", command }, {
|
||||
onOpen: () => listener.onOpen(),
|
||||
onText: (line) => listener.onLine(line),
|
||||
onStderr: (line) => listener.onStderr(line),
|
||||
onClosed: (info) => listener.onClosed(info.exitStatus ?? null),
|
||||
onFailure: (failure) => {
|
||||
onSshFailure?.(failure);
|
||||
listener.onFailure(describeSshFailure(failure));
|
||||
},
|
||||
});
|
||||
return new TmuxClient({ openChannel, label, cols, rows, stateKey: role === "control" ? connectionId : "" });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sshPasswordSecretId(connectionId: string): string {
|
||||
return `terminal.password.${connectionId}`;
|
||||
}
|
||||
|
||||
/** Failures that retrying can't fix: the user has to verify a key or supply credentials. */
|
||||
function sshFailureNeedsUser(failure: SshFailure): boolean {
|
||||
return failure.kind === "hostkey-unknown" || failure.kind === "hostkey-mismatch" || failure.kind === "auth";
|
||||
}
|
||||
|
||||
/**
|
||||
* A control's SSH channel failed. If the user has to act, remember why, and
|
||||
* if they're looking at the hub's lists, open the page that fixes it.
|
||||
*/
|
||||
function noteSshFailure(control: ControlConnection, failure: SshFailure): void {
|
||||
if (!sshFailureNeedsUser(failure) || controls.get(control.config.id) !== control || !control.client) return;
|
||||
control.sshIssue = failure;
|
||||
renderHubWindows();
|
||||
for (const window of windows.values()) {
|
||||
if (window.kind === "hub" && window.foreground && (window.mode === "sessions" || window.mode === "connections")) {
|
||||
openSshIssuePage(window, control);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile the live control clients with the stored connection list: start
|
||||
* enabled connections, stop disabled or removed ones, restart ones whose
|
||||
@@ -663,6 +838,7 @@ function syncControlsFromSettings(): void {
|
||||
control = {
|
||||
config: connection,
|
||||
client: null,
|
||||
sshIssue: null,
|
||||
state: null,
|
||||
unsubscribers: [],
|
||||
reconnectTimer: null,
|
||||
@@ -684,12 +860,13 @@ function syncControlsFromSettings(): void {
|
||||
|
||||
function startControl(control: ControlConnection): void {
|
||||
stopControl(control);
|
||||
const options = clientOptionsFor(control.config);
|
||||
if (!options) {
|
||||
const recipe = clientOptionsFor(control.config);
|
||||
if (!recipe) {
|
||||
control.state = null;
|
||||
return;
|
||||
}
|
||||
const client = new G2MirrorClient(options);
|
||||
const grid = viewGrid();
|
||||
const client = recipe.create(grid.cols, grid.rows, "control", (failure) => noteSshFailure(control, failure));
|
||||
const connectionId = control.config.id;
|
||||
control.client = client;
|
||||
control.state = client.state();
|
||||
@@ -703,6 +880,7 @@ function startControl(control: ControlConnection): void {
|
||||
control.state = state;
|
||||
if (state.phase === "connected") {
|
||||
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
|
||||
control.sshIssue = null;
|
||||
maybeCacheServerName(control, state.serverName);
|
||||
}
|
||||
if (state.phase === "failed") scheduleControlReconnect(control);
|
||||
@@ -777,6 +955,8 @@ function scheduleControlReconnect(control: ControlConnection): void {
|
||||
if (!terminalAutoReconnectSetting.get()) return;
|
||||
if (windows.size === 0 && !hasTerminalBackgroundWork()) return;
|
||||
if (!control.config.enabled) return;
|
||||
// Waiting on the user (host key, password) or on the host (no usable tmux).
|
||||
if (control.sshIssue || control.state?.retryable === false) return;
|
||||
const delayMs = control.reconnectDelayMs;
|
||||
control.reconnectDelayMs = Math.min(control.reconnectDelayMs * 2, RECONNECT_MAX_DELAY_MS);
|
||||
control.reconnectTimer = setTimeout(() => {
|
||||
@@ -943,7 +1123,7 @@ function createViewWindow(
|
||||
const cell = getTerminalFontConfig();
|
||||
const gridCols = Math.floor(viewport.width / cell.cellWidth);
|
||||
const gridRows = Math.floor(viewport.height / cell.cellHeight);
|
||||
const client = new G2MirrorClient({ ...view.options, cols: gridCols, rows: gridRows });
|
||||
const client = view.options.create(gridCols, gridRows, "view");
|
||||
const window: ViewWindow = {
|
||||
kind: "view",
|
||||
hub: null,
|
||||
@@ -1238,6 +1418,11 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "page") {
|
||||
window.page?.onBack();
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "connections") {
|
||||
setHubMode(window, "sessions");
|
||||
renderAndSubmit(window, frameId);
|
||||
@@ -1260,6 +1445,35 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal view ignored input");
|
||||
}
|
||||
|
||||
/**
|
||||
* A key from a hardware keyboard (the glasses' keyboard bridge) that no
|
||||
* keybinding claimed. A session with nothing over it gets the key as its
|
||||
* terminal would: Escape and the arrows belong to the program, and the way
|
||||
* out is Escape with Shift, Ctrl or Alt, which never arrives here (it is a
|
||||
* built-in keybinding for "back"). Anywhere else (the hub's lists, an open
|
||||
* menu) the key is handed back, to act as navigation.
|
||||
*/
|
||||
function handleKey(window: TerminalWindow, event: KeyEvent): void {
|
||||
if (window.kind === "view" && !window.menu?.isOpen()) {
|
||||
activeViewId = window.windowId;
|
||||
const page = terminalScrollbackKey(event);
|
||||
if (page !== null) {
|
||||
handleViewScroll(window, page, 0);
|
||||
return;
|
||||
}
|
||||
const bytes = encodeTerminalKey(event, window.emulator.applicationCursorKeys());
|
||||
if (bytes === null) return;
|
||||
// Typing follows the live screen, as in any terminal emulator.
|
||||
if (window.scrollTop !== null) {
|
||||
window.scrollTop = null;
|
||||
scheduleRender(window);
|
||||
}
|
||||
window.client.sendInput(bytes);
|
||||
return;
|
||||
}
|
||||
post({ type: "key-unhandled", windowId: window.windowId, event });
|
||||
}
|
||||
|
||||
/** Top visible absolute line index when following the live bottom. */
|
||||
function followTop(window: ViewWindow): number {
|
||||
return window.historyNext + window.emulator.bufferLength() - window.gridRows;
|
||||
@@ -1386,6 +1600,154 @@ function setHubMode(window: HubWindow, mode: HubMode): void {
|
||||
window.addError = "";
|
||||
}
|
||||
|
||||
function openHubPage(window: HubWindow, page: HubPage): void {
|
||||
if (window.mode === "add") endAddConnection(window);
|
||||
if (window.mode === "page") window.page?.onLeave?.();
|
||||
window.page = page;
|
||||
window.mode = "page";
|
||||
scheduleRender(window);
|
||||
}
|
||||
|
||||
/** Leave the current page (if any) for `mode`. */
|
||||
function closeHubPage(window: HubWindow, mode: HubMode): void {
|
||||
const page = window.page;
|
||||
window.page = null;
|
||||
page?.onLeave?.();
|
||||
setHubMode(window, mode);
|
||||
scheduleRender(window);
|
||||
}
|
||||
|
||||
/** The list a page returns to: the one it was opened from. */
|
||||
function hubListMode(window: HubWindow): HubMode {
|
||||
return window.mode === "connections" ? "connections" : "sessions";
|
||||
}
|
||||
|
||||
function openSshIssuePage(window: HubWindow, control: ControlConnection): void {
|
||||
const issue = control.sshIssue;
|
||||
if (!issue) return;
|
||||
if (issue.kind === "hostkey-unknown" && issue.hostKey) {
|
||||
openHostKeyPage(window, control, issue.hostKey);
|
||||
} else if (issue.kind === "hostkey-mismatch" && issue.hostKey) {
|
||||
openHostKeyChangedPage(window, control, issue.hostKey);
|
||||
} else if (issue.kind === "auth" && issue.passwordAllowed) {
|
||||
openPasswordPage(window, control, Boolean(issue.passwordTried));
|
||||
} else if (issue.kind === "auth") {
|
||||
openClientKeyPage(window, `${connectionDisplayName(control.config)} didn't accept this device's key (and doesn't take passwords).\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function hostKeyTypeName(type: string): string {
|
||||
if (type === "ssh-ed25519") return "ED25519";
|
||||
if (type.startsWith("ecdsa-")) return "ECDSA";
|
||||
if (type === "ssh-rsa") return "RSA";
|
||||
return type;
|
||||
}
|
||||
|
||||
function hostKeyFile(type: string): string {
|
||||
if (type === "ssh-ed25519") return "/etc/ssh/ssh_host_ed25519_key.pub";
|
||||
if (type.startsWith("ecdsa-")) return "/etc/ssh/ssh_host_ecdsa_key.pub";
|
||||
return "/etc/ssh/ssh_host_rsa_key.pub";
|
||||
}
|
||||
|
||||
/** First connection to an SSH host: show the key's fingerprint; click trusts it. */
|
||||
function openHostKeyPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
|
||||
const returnMode = hubListMode(window);
|
||||
openHubPage(window, {
|
||||
title: "Verify host key",
|
||||
body: () =>
|
||||
`First connection to ${connectionStringHostLabel(control.config.url)}. Its ${hostKeyTypeName(hostKey.type)} host key fingerprint is:\n` +
|
||||
`${hostKey.fingerprint}\n\n` +
|
||||
`To check it, run this on the host and compare:\nssh-keygen -lf ${hostKeyFile(hostKey.type)}`,
|
||||
hint: `click trust and connect ${GESTURE_DOUBLE_CLICK} cancel`,
|
||||
onClick: () => {
|
||||
const hostKeys = [...control.config.hostKeys.filter((key) => key !== hostKey.key), hostKey.key];
|
||||
control.config.hostKeys = hostKeys;
|
||||
updateConnection(control.config.id, { hostKeys });
|
||||
control.sshIssue = null;
|
||||
closeHubPage(window, returnMode);
|
||||
connectControl(control);
|
||||
},
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
function openHostKeyChangedPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
|
||||
const returnMode = hubListMode(window);
|
||||
openHubPage(window, {
|
||||
title: "HOST KEY CHANGED",
|
||||
body: () =>
|
||||
`${connectionStringHostLabel(control.config.url)} presented a different host key than the one trusted before: ` +
|
||||
`${hostKeyTypeName(hostKey.type)} ${hostKey.fingerprint}\n\n` +
|
||||
"Someone may be intercepting the connection, so Faceclaw won't connect. If the host was reinstalled or its keys " +
|
||||
'were regenerated, use "Forget host key" on this connection in Manage Connections, then verify the new fingerprint.',
|
||||
hint: `${GESTURE_DOUBLE_CLICK} back`,
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
/** Ask for the host's password on the phone (masked editor); click saves it and connects. */
|
||||
function openPasswordPage(window: HubWindow, control: ControlConnection, rejected: boolean): void {
|
||||
const returnMode = hubListMode(window);
|
||||
terminalSshPasswordSetting.set("");
|
||||
post({ type: "start-text-setting-edit", settingId: terminalSshPasswordSetting.id });
|
||||
openHubPage(window, {
|
||||
title: "SSH password",
|
||||
body: () => {
|
||||
const typed = terminalSshPasswordSetting.get();
|
||||
return (
|
||||
(rejected ? "The host refused the saved password.\n" : "") +
|
||||
`Type the password for ${connectionStringHostLabel(control.config.url)} in the phone app, then click to connect. ` +
|
||||
"It is saved encrypted on this phone.\n\n" +
|
||||
(typed ? "*".repeat(Math.min(typed.length, 32)) : "(nothing typed yet)")
|
||||
);
|
||||
},
|
||||
hint: `click connect ${GESTURE_DOUBLE_CLICK} cancel`,
|
||||
onClick: () => {
|
||||
const password = terminalSshPasswordSetting.get();
|
||||
if (!password) return;
|
||||
setSshSecret(sshPasswordSecretId(control.config.id), password);
|
||||
control.sshIssue = null;
|
||||
closeHubPage(window, returnMode);
|
||||
connectControl(control);
|
||||
},
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
onLeave: () => {
|
||||
terminalSshPasswordSetting.set("");
|
||||
post({ type: "end-text-setting-edit" });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** This device's SSH public key (the private half never leaves the keystore); click copies it. */
|
||||
function openClientKeyPage(window: HubWindow, intro = ""): void {
|
||||
const returnMode = hubListMode(window);
|
||||
let key: { authorizedKey: string; fingerprint: string } | null = null;
|
||||
let error = "";
|
||||
try {
|
||||
key = sshClientKey(SSH_KEY_COMMENT);
|
||||
} catch (caught) {
|
||||
error = String((caught as Error)?.message ?? caught);
|
||||
}
|
||||
let copied = false;
|
||||
openHubPage(window, {
|
||||
title: "This device's SSH key",
|
||||
body: () =>
|
||||
key
|
||||
? `${intro}Add this line to ~/.ssh/authorized_keys on each host to log in with this phone's key:\n` +
|
||||
`${key.authorizedKey}\n${key.fingerprint}` +
|
||||
(copied ? "\n\nCopied to the phone's clipboard." : "")
|
||||
: `Couldn't create this device's key: ${error}`,
|
||||
hint: key ? `click copy to phone clipboard ${GESTURE_DOUBLE_CLICK} back` : `${GESTURE_DOUBLE_CLICK} back`,
|
||||
onClick: key
|
||||
? () => {
|
||||
copyToPhoneClipboard("SSH public key", key!.authorizedKey);
|
||||
copied = true;
|
||||
}
|
||||
: undefined,
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One control connection's sessions in this hub window's display order:
|
||||
* most-recently-updated first as of when the window became visible, with
|
||||
@@ -1441,6 +1803,7 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
];
|
||||
const multiHost = controls.size > 1;
|
||||
const canLaunch = launchPresetNames().length > 0;
|
||||
const isTmux = (control: ControlConnection) => clientOptionsFor(control.config)?.kind === "tmux";
|
||||
// Every configured host is listed under its own heading, in stored order,
|
||||
// whatever its connection state: a connected host lists its sessions, any
|
||||
// other shows its state on the heading plus a Connect row (unless it is
|
||||
@@ -1450,10 +1813,17 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
const name = connectionDisplayName(control.config);
|
||||
if (!isControlConnected(control)) {
|
||||
items.push({ label: `${name} (${controlStatusWord(control)})`, heading: true });
|
||||
if (control.sshIssue && control.config.enabled) {
|
||||
items.push({ label: sshIssueAction(control.sshIssue), onSelect: () => openSshIssuePage(window, control) });
|
||||
continue;
|
||||
}
|
||||
const connecting = control.config.enabled && control.state?.phase === "connecting";
|
||||
if (!connecting && clientOptionsFor(control.config)) {
|
||||
// A failure retrying can't fix (tmux missing or too old) says why;
|
||||
// selecting it retries once the host is fixed.
|
||||
const unfixable = control.config.enabled && control.state?.retryable === false;
|
||||
items.push({
|
||||
label: control.reconnectTimer ? "Connect now" : "Connect",
|
||||
label: unfixable ? control.state!.status : control.reconnectTimer ? "Connect now" : "Connect",
|
||||
onSelect: () => connectControl(control),
|
||||
});
|
||||
}
|
||||
@@ -1462,7 +1832,8 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
items.push({
|
||||
label: name,
|
||||
heading: true,
|
||||
onSelect: canLaunch ? () => launchOnHost(window, control) : undefined,
|
||||
// tmux hosts can always start a shell, presets or not.
|
||||
onSelect: canLaunch || isTmux(control) ? () => launchOnHost(window, control) : undefined,
|
||||
});
|
||||
const sessions = orderedSessions(window, control);
|
||||
for (const session of sessions) {
|
||||
@@ -1478,7 +1849,11 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
}
|
||||
if (!sessions.length) {
|
||||
items.push({
|
||||
label: multiHost ? "(no live sessions)" : "(no live sessions; run g2mirror <command>)",
|
||||
label: isTmux(control)
|
||||
? "(no tmux sessions; select the host to start one)"
|
||||
: multiHost
|
||||
? "(no live sessions)"
|
||||
: "(no live sessions; run g2mirror <command>)",
|
||||
onSelect: () => control.client?.listSessions(),
|
||||
});
|
||||
}
|
||||
@@ -1499,6 +1874,10 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
|
||||
label: "Add connection",
|
||||
onSelect: () => beginAddConnection(window),
|
||||
},
|
||||
{
|
||||
label: "This device's SSH key",
|
||||
onSelect: () => openClientKeyPage(window),
|
||||
},
|
||||
{
|
||||
label: "Done",
|
||||
onSelect: () => setHubMode(window, "sessions"),
|
||||
@@ -1509,6 +1888,13 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
|
||||
|
||||
function controlStatusWord(control: ControlConnection): string {
|
||||
if (!control.config.enabled) return "off";
|
||||
const issue = control.sshIssue;
|
||||
if (issue) {
|
||||
if (issue.kind === "hostkey-unknown") return "verify host key";
|
||||
if (issue.kind === "hostkey-mismatch") return "HOST KEY CHANGED";
|
||||
if (issue.passwordTried) return "password rejected";
|
||||
return issue.passwordAllowed ? "password needed" : "key not authorized";
|
||||
}
|
||||
switch (control.state?.phase) {
|
||||
case "connected":
|
||||
case "attached":
|
||||
@@ -1541,6 +1927,32 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
|
||||
connectControl(control);
|
||||
},
|
||||
},
|
||||
...(control.config.hostKeys.length
|
||||
? [
|
||||
{
|
||||
label: "Forget host key",
|
||||
onSelect: (ctx: { stack: { pop(): unknown } }) => {
|
||||
ctx.stack.pop();
|
||||
control.config.hostKeys = [];
|
||||
updateConnection(control.config.id, { hostKeys: [] });
|
||||
control.sshIssue = null;
|
||||
if (control.config.enabled) startControl(control);
|
||||
renderHubWindows();
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(isSshConnection(control) && getSshSecret(sshPasswordSecretId(control.config.id))
|
||||
? [
|
||||
{
|
||||
label: "Forget saved password",
|
||||
onSelect: (ctx: { stack: { pop(): unknown } }) => {
|
||||
ctx.stack.pop();
|
||||
removeSshSecret(sshPasswordSecretId(control.config.id));
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
label: `Remove ${name}`,
|
||||
onSelect: (ctx) => {
|
||||
@@ -1558,6 +1970,11 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
|
||||
windowMenu(window).open(items);
|
||||
}
|
||||
|
||||
function isSshConnection(control: ControlConnection): boolean {
|
||||
const kind = clientOptionsFor(control.config)?.kind;
|
||||
return kind === "tmux" || kind === "ssh-g2mirror";
|
||||
}
|
||||
|
||||
/** Enable and (re)connect now; also the manual retry path, so reset backoff. */
|
||||
function connectControl(control: ControlConnection): void {
|
||||
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
|
||||
@@ -1578,6 +1995,7 @@ function disconnectControl(control: ControlConnection): void {
|
||||
|
||||
function removeControl(control: ControlConnection): void {
|
||||
stopControl(control);
|
||||
removeSshSecret(sshPasswordSecretId(control.config.id));
|
||||
controls.delete(control.config.id);
|
||||
saveConnections(loadConnections().filter((connection) => connection.id !== control.config.id));
|
||||
renderHubWindows();
|
||||
@@ -1586,7 +2004,7 @@ function removeControl(control: ControlConnection): void {
|
||||
/**
|
||||
* Enter the Add-connection screen: clear the draft and ask the shell to open
|
||||
* the phone text editor on it. The user types (or voice-inputs) the
|
||||
* g2mirror:// string, then clicks to save or double-clicks to cancel.
|
||||
* connection string, then clicks to save or double-clicks to cancel.
|
||||
*/
|
||||
function beginAddConnection(window: HubWindow): void {
|
||||
terminalNewConnectionSetting.set("");
|
||||
@@ -1596,15 +2014,15 @@ function beginAddConnection(window: HubWindow): void {
|
||||
|
||||
function saveAddConnection(window: HubWindow): void {
|
||||
const url = terminalNewConnectionSetting.get();
|
||||
if (!parseConnectionString(url)) {
|
||||
if (!parseAnyConnectionString(url)) {
|
||||
window.addError = url
|
||||
? "Not a g2mirror://token@host connection string."
|
||||
: "Enter a g2mirror:// connection string first.";
|
||||
? "Not an ssh://user@host or g2mirror://token@host string."
|
||||
: "Enter a connection string first.";
|
||||
scheduleRender(window);
|
||||
return;
|
||||
}
|
||||
const connections = loadConnections();
|
||||
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true });
|
||||
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true, hostKeys: [] });
|
||||
saveConnections(connections);
|
||||
endAddConnection(window);
|
||||
setHubMode(window, "connections");
|
||||
@@ -1634,6 +2052,15 @@ function handleHubInput(window: HubWindow, event: InputEvent, frameId: number):
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal add-connection ignored input");
|
||||
return;
|
||||
}
|
||||
if (window.mode === "page") {
|
||||
if (event.type === "click" && window.page?.onClick) {
|
||||
window.page.onClick();
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal page ignored input");
|
||||
return;
|
||||
}
|
||||
const list = hubMenu(window);
|
||||
list.setItems(hubItems(window));
|
||||
switch (event.type) {
|
||||
@@ -1800,6 +2227,7 @@ function openViewWindow(control: ControlConnection, socket: string, label: strin
|
||||
focus: true,
|
||||
// Default to tall sessions; the app display setting can override this.
|
||||
heightMode: "max",
|
||||
acceptsKeys: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1820,6 +2248,11 @@ function launchPresetNames(): string[] {
|
||||
*/
|
||||
function launchOnHost(window: HubWindow, control: ControlConnection): void {
|
||||
const presets = launchPresetNames();
|
||||
if (presets.length === 0) {
|
||||
// A tmux host with no presets configured: a new session running the login shell.
|
||||
launchFromUi(control, "shell");
|
||||
return;
|
||||
}
|
||||
if (presets.length === 1) {
|
||||
launchFromUi(control, presets[0]!);
|
||||
return;
|
||||
@@ -1864,6 +2297,9 @@ function paintHub(window: HubWindow): GrayImage {
|
||||
if (window.mode === "add") {
|
||||
return paintAddConnection(window);
|
||||
}
|
||||
if (window.mode === "page" && window.page) {
|
||||
return paintHubPage(window, window.page);
|
||||
}
|
||||
const { viewportWidth: width, viewportHeight: height, padding } = window;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const font = chromeFont();
|
||||
@@ -1878,7 +2314,7 @@ function paintHub(window: HubWindow): GrayImage {
|
||||
|
||||
let listTop = header.listTop;
|
||||
if (window.mode === "sessions" && controls.size === 0) {
|
||||
image.drawText(font, header.titleX, listTop, "Add a g2mirror:// connection to get started, see:", 150);
|
||||
image.drawText(font, header.titleX, listTop, "Add an ssh:// (tmux) or g2mirror:// connection to get started, see:", 150);
|
||||
image.drawText(font, header.titleX, listTop + step, "https://github.com/jimrandomh/g2mirror", 190);
|
||||
listTop += 2 * step + 6;
|
||||
}
|
||||
@@ -1920,8 +2356,8 @@ function paintAddConnection(window: HubWindow): GrayImage {
|
||||
image.drawText(font, padding.left, padding.top, "Add connection", 220);
|
||||
const bodyX = padding.left + PAGE_BODY_INDENT;
|
||||
const instructionsY = padding.top + 10 + step;
|
||||
image.drawText(font, bodyX, instructionsY, "Type the g2mirror://token@host string in the", 170);
|
||||
image.drawText(font, bodyX, instructionsY + step, "phone app (or use voice input from the menu).", 170);
|
||||
image.drawText(font, bodyX, instructionsY, "Type ssh://user@host (tmux) or g2mirror://token@host in", 170);
|
||||
image.drawText(font, bodyX, instructionsY + step, "the phone app (or use voice input from the menu).", 170);
|
||||
const draftY = instructionsY + 3 * step;
|
||||
const draft = terminalNewConnectionSetting.get();
|
||||
image.drawText(font, bodyX, draftY, truncateLabel(draft || "(empty)", 52), 220);
|
||||
@@ -1932,6 +2368,34 @@ function paintAddConnection(window: HubWindow): GrayImage {
|
||||
return image;
|
||||
}
|
||||
|
||||
/** A page of text: title at the top padding, wrapped body, hint on the bottom padding. */
|
||||
function paintHubPage(window: HubWindow, page: HubPage): GrayImage {
|
||||
const { viewportWidth: width, viewportHeight: height, padding } = window;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const font = chromeFont();
|
||||
const step = lineStep(font);
|
||||
image.drawText(font, padding.left, padding.top, page.title, 220);
|
||||
const bodyX = padding.left + PAGE_BODY_INDENT;
|
||||
const hintY = height - padding.bottom - font.lineHeight;
|
||||
const lines = wrapText(font, page.body(), Math.max(1, width - padding.right - bodyX), { breakLongWords: true });
|
||||
let y = padding.top + 10 + step;
|
||||
for (const line of lines) {
|
||||
if (y + step > hintY) break;
|
||||
image.drawText(font, bodyX, y, line, 190);
|
||||
y += step;
|
||||
}
|
||||
image.drawText(font, bodyX, hintY, page.hint, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
/** The action row under a host whose SSH connection needs the user. */
|
||||
function sshIssueAction(issue: SshFailure): string {
|
||||
if (issue.kind === "hostkey-unknown") return "Verify host key...";
|
||||
if (issue.kind === "hostkey-mismatch") return "Host key changed!";
|
||||
if (issue.passwordAllowed) return issue.passwordTried ? "Re-enter password..." : "Enter password...";
|
||||
return "Show this device's SSH key...";
|
||||
}
|
||||
|
||||
function truncateLabel(text: string, maxLength: number): string {
|
||||
return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import type {
|
||||
G2MirrorHistoryReply,
|
||||
G2MirrorPhase,
|
||||
G2MirrorSession,
|
||||
G2MirrorState,
|
||||
} from "../../native/terminal-session-types";
|
||||
|
||||
/**
|
||||
* What the Terminal worker needs from a session backend. G2MirrorClient (a
|
||||
* g2mirror server, directly or tunnelled over SSH) and TmuxClient (tmux
|
||||
* control mode over SSH) both provide it. The model is g2mirror's: a client
|
||||
* starts "connecting", reaches "connected" (it can list and launch sessions,
|
||||
* and reports bells/activity/titles for them), and can then attach to one
|
||||
* session, view it (snapshot + live output), page its scrollback, and type
|
||||
* into it.
|
||||
*/
|
||||
export interface TerminalClient {
|
||||
state(): G2MirrorState;
|
||||
onStateChange(listener: (state: G2MirrorState) => void): () => void;
|
||||
/** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
|
||||
onTerminalData(listener: (data: Uint8Array, kind: "snapshot" | "output") => void): () => void;
|
||||
onSessionAttached(listener: (command: string) => void): () => void;
|
||||
onSessionDetached(listener: (reason: string) => void): () => void;
|
||||
onBell(listener: (socket: string, lastBellAtMs: number) => void): () => void;
|
||||
onActivity(listener: (socket: string, lastOutputAtMs: number) => void): () => void;
|
||||
onTitle(listener: (socket: string, title: string) => void): () => void;
|
||||
/** Fired after a snapshot's terminal data, with its scrollback splice point. */
|
||||
onSnapshot(listener: (historyNext: number, historyOldest: number) => void): () => void;
|
||||
onHistoryLines(listener: (reply: G2MirrorHistoryReply) => void): () => void;
|
||||
setViewport(cols: number, rows: number): void;
|
||||
start(): void;
|
||||
stop(): void;
|
||||
listSessions(): void;
|
||||
launchSession(preset: string): Promise<string>;
|
||||
connectSession(socket: string): void;
|
||||
disconnectSession(): void;
|
||||
view(): void;
|
||||
unview(): void;
|
||||
requestHistory(before: number, limit?: number): void;
|
||||
sendInput(text: string): void;
|
||||
submitInput(text: string): void;
|
||||
}
|
||||
|
||||
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
|
||||
@@ -82,4 +82,9 @@ export class TerminalEmulator {
|
||||
cursorCol(): number {
|
||||
return this.term.buffer.active.cursorX;
|
||||
}
|
||||
|
||||
/** DECCKM: the session asked for application cursor keys (ESC O A rather than ESC [ A). */
|
||||
applicationCursorKeys(): boolean {
|
||||
return this.term.modes.applicationCursorKeysMode;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* What a terminal emulator sends for a key from a hardware keyboard, by the
|
||||
* classic xterm conventions: Ctrl chords become control characters, Alt
|
||||
* prefixes ESC, and modified cursor and function keys carry the xterm
|
||||
* modifier parameter (1 + shift + 2*alt + 4*ctrl). Follows the Glassly
|
||||
* Terminal miniapp's encoder (terminal/keys.ts, MIT).
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
|
||||
const ESC = "\x1b";
|
||||
|
||||
/** Keys xterm encodes as `ESC [ <n> ~`, with a modifier as `;m`. */
|
||||
const TILDE_KEYS: Record<string, number> = {
|
||||
Insert: 2,
|
||||
Delete: 3,
|
||||
PageUp: 5,
|
||||
PageDown: 6,
|
||||
F5: 15,
|
||||
F6: 17,
|
||||
F7: 18,
|
||||
F8: 19,
|
||||
F9: 20,
|
||||
F10: 21,
|
||||
F11: 23,
|
||||
F12: 24,
|
||||
};
|
||||
|
||||
/** Keys with a single-letter final byte: SS3 in application cursor mode, CSI otherwise. */
|
||||
const LETTER_KEYS: Record<string, string> = {
|
||||
ArrowUp: "A",
|
||||
ArrowDown: "B",
|
||||
ArrowRight: "C",
|
||||
ArrowLeft: "D",
|
||||
Home: "H",
|
||||
End: "F",
|
||||
};
|
||||
|
||||
/** F1-F4 are always SS3 (`ESC O P`), or CSI with a modifier. */
|
||||
const SS3_KEYS: Record<string, string> = { F1: "P", F2: "Q", F3: "R", F4: "S" };
|
||||
|
||||
/** Ctrl with a punctuation or digit key, per xterm (^[ ^\ ^] ^^ ^_ ^? ^@). */
|
||||
const CONTROL_PUNCTUATION: Record<string, string> = {
|
||||
"[": "\x1b",
|
||||
"\\": "\x1c",
|
||||
"]": "\x1d",
|
||||
"^": "\x1e",
|
||||
"6": "\x1e",
|
||||
"_": "\x1f",
|
||||
"-": "\x1f",
|
||||
"?": "\x7f",
|
||||
"/": "\x7f",
|
||||
"8": "\x7f",
|
||||
"@": "\x00",
|
||||
"2": "\x00",
|
||||
};
|
||||
|
||||
/**
|
||||
* The text to write to the terminal for one key event, or null when the key
|
||||
* sends nothing: releases, bare modifiers, Meta chords, and keys a terminal
|
||||
* has no bytes for (volume, media). `appCursor` is DECCKM, which the emulator
|
||||
* tracks from the session's output.
|
||||
*/
|
||||
export function encodeTerminalKey(event: KeyEvent, appCursor: boolean): string | null {
|
||||
if (event.phase !== "down") return null;
|
||||
const { key, modifiers } = event;
|
||||
if (modifiers.meta) return null;
|
||||
const modifierParam = 1 + (modifiers.shift ? 1 : 0) + (modifiers.alt ? 2 : 0) + (modifiers.ctrl ? 4 : 0);
|
||||
const modified = (prefix: string, final: string): string | null =>
|
||||
modifierParam > 1 ? `${ESC}[${prefix};${modifierParam}${final}` : null;
|
||||
|
||||
switch (key) {
|
||||
case "Enter":
|
||||
return modifiers.alt ? `${ESC}\r` : "\r";
|
||||
case "Escape":
|
||||
return ESC;
|
||||
case "Tab":
|
||||
return modifiers.shift ? `${ESC}[Z` : "\t";
|
||||
case "Backspace":
|
||||
if (modifiers.alt) return `${ESC}\x7f`;
|
||||
return modifiers.ctrl ? "\x08" : "\x7f";
|
||||
case " ":
|
||||
return modifiers.ctrl ? "\x00" : modifiers.alt ? `${ESC} ` : " ";
|
||||
}
|
||||
|
||||
const letter = LETTER_KEYS[key];
|
||||
if (letter) return modified("1", letter) ?? (appCursor ? `${ESC}O${letter}` : `${ESC}[${letter}`);
|
||||
const tilde = TILDE_KEYS[key];
|
||||
if (tilde) return modified(String(tilde), "~") ?? `${ESC}[${tilde}~`;
|
||||
const ss3 = SS3_KEYS[key];
|
||||
if (ss3) return modified("1", ss3) ?? `${ESC}O${ss3}`;
|
||||
|
||||
if (modifiers.ctrl) {
|
||||
// From the unshifted key: ^A..^Z, then the punctuation set.
|
||||
const base = key.length === 1 ? key.toLowerCase() : "";
|
||||
let char: string | undefined;
|
||||
if (base >= "a" && base <= "z") char = String.fromCharCode(base.charCodeAt(0) - 96);
|
||||
else char = CONTROL_PUNCTUATION[key];
|
||||
if (char === undefined) return null;
|
||||
return modifiers.alt ? `${ESC}${char}` : char;
|
||||
}
|
||||
|
||||
// With Alt held the translator leaves `text` out (it is a chord, not
|
||||
// typing); a terminal still wants ESC and the character.
|
||||
const text = event.text ?? (modifiers.alt && key.length === 1 ? key : undefined);
|
||||
if (text === undefined) return null;
|
||||
return modifiers.alt ? `${ESC}${text}` : text;
|
||||
}
|
||||
|
||||
/** Shift+PageUp / Shift+PageDown page the emulator's own scrollback, as in xterm. */
|
||||
export function terminalScrollbackKey(event: KeyEvent): -1 | 1 | null {
|
||||
if (event.phase !== "down") return null;
|
||||
const { shift, ctrl, alt, meta } = event.modifiers;
|
||||
if (!shift || ctrl || alt || meta) return null;
|
||||
if (event.key === "PageUp") return -1;
|
||||
if (event.key === "PageDown") return 1;
|
||||
return null;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,32 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
|
||||
import { type AppContext, type AppDefinition } from "../app-definition";
|
||||
import {
|
||||
createTranscribeAppWindow,
|
||||
createTranscribeReplayWindow,
|
||||
TRANSCRIBE_SURFACE_ID,
|
||||
TRANSCRIBE_WINDOW_ID,
|
||||
} from "./transcribe-app";
|
||||
|
||||
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
|
||||
let nextReplaySerial = 1;
|
||||
|
||||
/**
|
||||
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
|
||||
* own. Returns false where recordings can't be replayed.
|
||||
*/
|
||||
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
|
||||
if (!ctx.actions.openVoiceReplay) return false;
|
||||
const windowId = `transcribe:replay:${nextReplaySerial++}`;
|
||||
void ctx
|
||||
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
|
||||
const window = createTranscribeReplayWindow(windowId, path, name, options);
|
||||
if (!window) throw new Error("recordings can't be replayed here");
|
||||
return window;
|
||||
})
|
||||
.catch((error) => {
|
||||
ctx.appendLog(`transcribe replay window failed: ${error}`);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const transcribeApp: AppDefinition = {
|
||||
appId: "transcribe",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
|
||||
export function formatReplayTime(ms: number, withHours = false): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const seconds = String(total % 60).padStart(2, "0");
|
||||
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
|
||||
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
|
||||
}
|
||||
|
||||
/** The time picker's fields, in display and click order. */
|
||||
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
|
||||
export type TimeField = (typeof TIME_FIELDS)[number];
|
||||
|
||||
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
|
||||
|
||||
/**
|
||||
* Step a position (whole seconds) by `delta` of a field's unit, carrying
|
||||
* into the neighboring fields, within [0, maxSeconds].
|
||||
*/
|
||||
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
|
||||
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
|
||||
import { CONTEXT_MENU_DIM } from "../../ui/menu";
|
||||
import { centeredTextY } from "../../ui/metrics";
|
||||
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
|
||||
|
||||
const PADDING = 12;
|
||||
const FIELD_GAP = 6;
|
||||
const FIELD_PAD_X = 6;
|
||||
/** Fill and outline of the active field (the Timers and date pickers' field style). */
|
||||
const ACTIVE_FILL = 35;
|
||||
const ACTIVE_STROKE = 130;
|
||||
|
||||
export type TimePickerOptions = {
|
||||
title: string;
|
||||
/** Where the picker starts, and the furthest it goes, in ms. */
|
||||
initialMs: number;
|
||||
maxMs: number;
|
||||
onPick: (positionMs: number) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* A modal position picker for a recording: hours (when the recording has
|
||||
* any), minutes and seconds, over the dimmed content beneath. Input follows
|
||||
* the date picker's: scroll adjusts the active field (up is later, carrying
|
||||
* into the fields beside it), click moves to the next field and on the last
|
||||
* one picks, double-click steps back a field and on the first one cancels.
|
||||
* A watch's swipes move between fields and adjust.
|
||||
*/
|
||||
export class TimePickerLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
readonly dimUnderneath = CONTEXT_MENU_DIM;
|
||||
private readonly fields: readonly TimeField[];
|
||||
private readonly maxSeconds: number;
|
||||
private seconds: number;
|
||||
private field = 0;
|
||||
|
||||
constructor(private readonly options: TimePickerOptions) {
|
||||
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
|
||||
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
|
||||
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "swipe-up":
|
||||
this.adjust(1);
|
||||
return;
|
||||
case "scroll-down":
|
||||
case "swipe-down":
|
||||
this.adjust(-1);
|
||||
return;
|
||||
case "swipe-right":
|
||||
this.field = Math.min(this.fields.length - 1, this.field + 1);
|
||||
return;
|
||||
case "swipe-left":
|
||||
this.field = Math.max(0, this.field - 1);
|
||||
return;
|
||||
case "click":
|
||||
if (this.field < this.fields.length - 1) {
|
||||
this.field++;
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
this.options.onPick(this.seconds * 1000);
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
if (this.field > 0) this.field--;
|
||||
else ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private adjust(delta: number): void {
|
||||
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
|
||||
}
|
||||
|
||||
private hints(field: number): string {
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, "adjust"],
|
||||
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
|
||||
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
|
||||
]);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const image = paintBelow();
|
||||
const small = getDefaultSmallFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const focused = ctx.stack.isFocused();
|
||||
const base = ctx.stack.getBaseSize();
|
||||
|
||||
const values = [
|
||||
String(Math.floor(this.seconds / 3600)),
|
||||
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
|
||||
String(this.seconds % 60).padStart(2, "0"),
|
||||
].slice(TIME_FIELDS.length - this.fields.length);
|
||||
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
|
||||
const fieldWidths = this.fields.map(
|
||||
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
|
||||
);
|
||||
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
|
||||
const boxHeight = medium.lineHeight + 6;
|
||||
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
|
||||
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
|
||||
const width = Math.min(
|
||||
base.width,
|
||||
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
|
||||
);
|
||||
const titleHeight = small.lineHeight + 12;
|
||||
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
|
||||
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
|
||||
const x = Math.max(0, Math.floor((base.width - width) / 2));
|
||||
const y = Math.max(0, Math.floor((base.height - height) / 2));
|
||||
|
||||
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
|
||||
image.fillRoundedRect(x, y, width, height, 1);
|
||||
image.drawRoundedRect(x, y, width, height, 72);
|
||||
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
|
||||
|
||||
const top = y + 8 + titleHeight;
|
||||
let fieldX = x + PADDING;
|
||||
for (let index = 0; index < this.fields.length; index++) {
|
||||
const fieldWidth = fieldWidths[index]!;
|
||||
const active = index === this.field;
|
||||
if (active) {
|
||||
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
|
||||
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
|
||||
}
|
||||
const value = values[index]!;
|
||||
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
|
||||
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
|
||||
const caption = this.fields[index]!;
|
||||
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
|
||||
top + boxHeight + 2, caption, active ? 190 : 110);
|
||||
fieldX += fieldWidth + FIELD_GAP;
|
||||
}
|
||||
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
|
||||
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
import { withRecordingIcon } from "../../ui/listening-header";
|
||||
import { TimePickerLayer } from "./time-picker";
|
||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
|
||||
}
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
// One timestamp per session names both the audio recording and a saved transcript.
|
||||
const stamp = transcriptTimestamp();
|
||||
const layer = new TranscribeLayer(stamp);
|
||||
// At launch, and again when the Microphones setting changes: a running
|
||||
// capture moves over to the microphones now asked for.
|
||||
const startCapture = () =>
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
},
|
||||
// The controller starts and stops the recording as the setting changes.
|
||||
toggleSettingMenuItem(saveRecordingSetting),
|
||||
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
app.requestRender();
|
||||
});
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
startCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
return app;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe in replay mode: a window of its own that transcribes a
|
||||
* recording (a .mka file from Save Recording) in place of the microphones,
|
||||
* in real time and through the same pipeline. It neither uses the mic nor
|
||||
* shows the mic tray icon, so it runs beside a live Transcribe window. The
|
||||
* header shows the position in the recording; a tap pauses and resumes; the
|
||||
* menu jumps to a time.
|
||||
*/
|
||||
export function createTranscribeReplayWindow(
|
||||
windowId: string,
|
||||
path: string,
|
||||
name: string,
|
||||
options: InProcessAppOptions,
|
||||
): InProcessWindow | null {
|
||||
const replay = options.actions.openVoiceReplay?.(path) ?? null;
|
||||
if (!replay) return null;
|
||||
// Names a saved transcript after the recording it came from.
|
||||
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId,
|
||||
title: name,
|
||||
iconLetter: "Tr",
|
||||
icon: "mic",
|
||||
closeable: true,
|
||||
// Relaunching Transcribe after a restart would open the microphone, not this recording.
|
||||
restoresApp: false,
|
||||
// Like a playing video: something to watch, not to touch.
|
||||
keepsScreenOn: () => replay.state === "playing",
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [
|
||||
{
|
||||
label: "Jump to time",
|
||||
// The length isn't known until the recording has been opened.
|
||||
disabled: () => replay.durationMs <= 0,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(
|
||||
new TimePickerLayer({
|
||||
title: "Jump to time",
|
||||
initialMs: replay.positionMs,
|
||||
maxMs: replay.durationMs,
|
||||
onPick: (positionMs) => {
|
||||
replay.jumpTo(positionMs);
|
||||
app.requestRender();
|
||||
},
|
||||
}),
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: replay.state === "paused" ? "Resume" : "Pause",
|
||||
disabled: () => replay.state !== "playing" && replay.state !== "paused",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.setPaused(replay.state === "playing");
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Restart from 0:00",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.jumpTo(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
},
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
replay.close();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// Started first, so the layer picks its opening status up as it subscribes.
|
||||
replay.start();
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
|
||||
import { type MicCount } from "../../ui/layers";
|
||||
import { micControlSupported } from "../microphones/mic-control";
|
||||
|
||||
export type TranscribeMics = "1" | "2" | "4";
|
||||
|
||||
const MICS_LABELS: Record<TranscribeMics, string> = {
|
||||
"1": "1 (mixed)",
|
||||
"2": "2 (one per temple)",
|
||||
"4": "4 (two per temple)",
|
||||
};
|
||||
|
||||
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
|
||||
export function separateMicsSupported(): boolean {
|
||||
return glassesTypeSetting.get() === "g2" && micControlSupported();
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones Transcribe captures. With more than
|
||||
* one, the temples stream their own microphones (each temple on its own
|
||||
* clock) instead of the glasses' mixed stream; a recording keeps every one,
|
||||
* and what is transcribed is a mix made on the phone.
|
||||
*/
|
||||
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
|
||||
id: "transcribe-mics",
|
||||
label: "Microphones",
|
||||
storageKey: "transcribe.mics",
|
||||
defaultValue: "1",
|
||||
values: ["1", "2", "4"],
|
||||
formatValue: (value) => MICS_LABELS[value],
|
||||
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
|
||||
description:
|
||||
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
|
||||
});
|
||||
|
||||
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
|
||||
export function transcribeMicCount(): MicCount {
|
||||
if (!separateMicsSupported()) return 1;
|
||||
return Number(transcribeMicsSetting.get()) as MicCount;
|
||||
}
|
||||
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { formatReplayTime } from "./replay-time";
|
||||
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*
|
||||
* Given a replay, it transcribes that recording instead of the microphone:
|
||||
* the header shows the position in the recording, a tap pauses and resumes,
|
||||
* and each jump leaves its time as a line in the transcript.
|
||||
*/
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeReplay: (() => void) | null = null;
|
||||
private readonly source: VoiceEventSource;
|
||||
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
/**
|
||||
* `stamp` names the saved transcript, matching the session's audio
|
||||
* recording. `replay` is the recording to transcribe in place of the
|
||||
* microphone; `replayName` is what to call it.
|
||||
*/
|
||||
constructor(
|
||||
private readonly stamp: string = transcriptTimestamp(),
|
||||
private readonly replay: VoiceReplay | null = null,
|
||||
private readonly replayName = "",
|
||||
) {
|
||||
this.source = replay ? replay.events : voiceControlBridge;
|
||||
if (replay) this.status = "Opening the recording...";
|
||||
}
|
||||
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.meter.start(requestRender, isVisible, this.source);
|
||||
this.unsubscribeTranscript = this.source.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
|
||||
// The current status arrives while subscribing, before the window has a
|
||||
// surface to render to; the window paints it when it is first shown.
|
||||
let subscribing = true;
|
||||
this.unsubscribeStatus = this.source.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
if (!subscribing) requestRender();
|
||||
});
|
||||
subscribing = false;
|
||||
const replay = this.replay;
|
||||
if (replay) {
|
||||
let shownSecond = -1;
|
||||
let shownState = replay.state;
|
||||
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
|
||||
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
|
||||
// The clock ticks in whole seconds; repaint when what it shows changed.
|
||||
const second = Math.floor(replay.positionMs / 1000);
|
||||
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
|
||||
shownSecond = second;
|
||||
shownState = replay.state;
|
||||
if (isVisible()) requestRender();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
|
||||
private replayStatus(replay: VoiceReplay): string {
|
||||
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
|
||||
const withHours = replay.durationMs >= 3_600_000;
|
||||
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
|
||||
if (replay.state === "paused") return `${clock} · Paused`;
|
||||
if (replay.state === "ended") return `${clock} · Finished`;
|
||||
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
|
||||
return this.listening ? clock : `${clock} · ${this.status}`;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
|
||||
const viewport = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(viewport.width, viewport.height, 0);
|
||||
|
||||
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const replay = this.replay;
|
||||
const status = replay
|
||||
? this.replayStatus(replay)
|
||||
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const playing = !replay || replay.state === "playing";
|
||||
const bodyTop = paintListeningHeader(
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
|
||||
voiceControlBridge.recording,
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
|
||||
this.source.recording,
|
||||
);
|
||||
const step = lineStep(font) + 2;
|
||||
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
|
||||
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
|
||||
|
||||
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
|
||||
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click" && this.replay) {
|
||||
this.replay.setPaused(this.replay.state === "playing");
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribeReplay?.();
|
||||
this.unsubscribeReplay = null;
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -36,6 +36,18 @@ export class TranscriptModel {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* A line of its own (where a replay jumped to), ending whatever was in
|
||||
* progress: what follows it starts a new paragraph.
|
||||
*/
|
||||
mark(label: string): void {
|
||||
const before = this.text;
|
||||
this.finalized = before ? `${before}\n${label}` : label;
|
||||
this.live = "";
|
||||
this.separator = "\n";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
|
||||
Reference in new issue
Block a user