[verified] merge upstream 25cb255b into all-features

This commit is contained in:
MasterControl committed 2026-11-19 22:21:26 +00:00
commit ffcdae660d
169 files changed
+16201 -597

No files matched your search

+6 -1
View File
@@ -55,8 +55,10 @@ export type GlanceboardProvider = {
isEnabled: () => boolean;
/** Whether a single tap shows the board (for tapTimeoutMs). */
showOnTap: () => boolean;
/** How long a tap or head-tilt keeps the board up. */
/** How long a tap keeps the board up (and a head-tilt, when its head-down is unlikely to come). */
tapTimeoutMs: () => number;
/** How long a head-tilt showing may last without a head-down (null = no limit). */
tiltTimeoutMs: () => number | null;
/** Whether a long-press holds the board up until released. */
showOnLongPress: () => boolean;
/** Whether the head-tilt wake goes to the board rather than the regular UI. */
@@ -187,6 +189,8 @@ export async function launchWorkerAppWindow(
matchExistingBy?: "appId" | "windowId";
/** Deliver watch swipes raw instead of the fallback (see WorkerWindowSpec). */
acceptsDirectional?: boolean;
/** Deliver hardware-keyboard keys as keys (see WorkerWindowSpec). */
acceptsKeys?: boolean;
},
): Promise<void> {
const existing = shell
@@ -207,6 +211,7 @@ export async function launchWorkerAppWindow(
icon: options.icon,
focus: true,
acceptsDirectional: options.acceptsDirectional,
acceptsKeys: options.acceptsKeys,
});
ctx.appendLog(`launched ${options.windowId}`);
}
+1
View File
@@ -312,6 +312,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+9 -2
View File
@@ -9,6 +9,7 @@ import { hasCalendarPermission } from "../../native/calendar-permissions";
import { type Layer, type LayerContext } from "../../ui/layers";
import { lineStep } from "../../ui/metrics";
import type { Viewport } from "../../ui/viewport";
import { listingDayMs, spanLabel, spansDays } from "./event-days";
/** How far a row's text sits inside its selection box; the title lines up with it. */
const ROW_TEXT_INSET = 10;
@@ -143,14 +144,20 @@ export class CalendarLayer implements Layer {
function buildEventRows(font: UiFont, events: CalendarEvent[]): EventRow[] {
const rows: EventRow[] = [];
let previousDayKey = "";
const nowMs = Date.now();
for (const event of events) {
const dayKey = dayKeyOf(event.startMs);
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(event.startMs);
// An event still going since an earlier day is listed under today, not its first day.
const listedMs = listingDayMs(event, nowMs);
const dayKey = dayKeyOf(listedMs);
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(listedMs);
previousDayKey = dayKey;
const lines: string[] = [];
const timeLabel = event.allDay ? "All day" : formatEventTime(event.startMs);
lines.push(`${timeLabel} ${event.title || "(untitled)"}`);
if (spansDays(event)) {
lines.push(spanLabel(event));
}
if (event.location) {
lines.push(event.location);
}
+43
View File
@@ -0,0 +1,43 @@
/**
* Which day an event is listed under, and how a multi-day span reads. Pure (no
* native or UI imports) so it can be unit-tested.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
export type DayEvent = { startMs: number; endMs: number; allDay: boolean };
function midnight(timestampMs: number): number {
const date = new Date(timestampMs);
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
}
/** The last moment the event covers: all-day ends are exclusive (the midnight after its last day). */
function lastMomentMs(event: DayEvent): number {
return event.allDay ? Math.max(event.startMs, event.endMs - 1) : Math.max(event.startMs, event.endMs);
}
/**
* The day to list an event under: its start, or today for one that began on an
* earlier day and is still going. Without this, the third day of a three-day
* event shows under its first day and never under today.
*/
export function listingDayMs(event: DayEvent, nowMs: number): number {
return Math.max(midnight(event.startMs), Math.min(midnight(nowMs), midnight(lastMomentMs(event))));
}
/** Whether the event covers more than one calendar day. */
export function spansDays(event: DayEvent): boolean {
return midnight(lastMomentMs(event)) > midnight(event.startMs);
}
/** "Mon 16 – Wed 18 Nov", or with both months when they differ ("Mon 30 Nov – Tue 1 Dec"). */
export function spanLabel(event: DayEvent): string {
const start = new Date(event.startMs);
const last = new Date(lastMomentMs(event));
const day = (date: Date) => `${WEEKDAYS[date.getDay()]} ${date.getDate()}`;
return start.getMonth() === last.getMonth() && start.getFullYear() === last.getFullYear()
? `${day(start)} – ${day(last)} ${MONTHS[last.getMonth()]}`
: `${day(start)} ${MONTHS[start.getMonth()]} – ${day(last)} ${MONTHS[last.getMonth()]}`;
}
+5 -3
View File
@@ -1,7 +1,7 @@
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import { clamp } from "../../util/numeric-util";
import { addImuListener, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
import { addImuListener, imuPitchDegrees, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
@@ -23,7 +23,8 @@ const BAR_RIGHT_INSET = 4;
/**
* Accelerometer demo: turns the IMU stream on while this page is visible (the
* Developer window is foregrounded and the screen is on), off otherwise, and
* shows the latest x/y/z reading with a simple per-axis bar.
* shows the latest x/y/z reading (Faceclaw's frame, imu-types.ts) with a
* simple per-axis bar, and the head's pitch.
*
* The IMU stream is experimental — the firmware may never populate it — so the
* page also reports how many samples have arrived.
@@ -92,7 +93,8 @@ export class AccelerometerDemoLayer implements Layer {
const bodyX = padding.left + BODY_INDENT;
image.drawText(small, padding.left, top, "Accelerometer", 220);
const status = this.enabled ? "IMU on" : "IMU off";
const pitch = this.reading ? `pitch ${Math.round(imuPitchDegrees(this.reading))}° ` : "";
const status = pitch + (this.enabled ? "IMU on" : "IMU off");
image.drawText(small, right - small.measureText(status), top, status, 150);
if (this.reading) {
+8
View File
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
import { AccelerometerDemoLayer } from "./accelerometer-demo";
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
import { LightSensorDemoLayer } from "./light-sensor-demo";
import { MouseTestLayer } from "./mouse-test";
import { MicrophonesMonitorLayer } from "./microphones-monitor";
import { ResourceUsageLayer } from "./resource-usage";
import { RingStatusLayer } from "./ring-status";
@@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
label: "Microphones",
onSelect: openMicrophones,
},
{
// The touchpad or mouse of a keyboard paired through the glasses.
label: "Mouse cursor",
onSelect: (ctx) => {
ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "BLE bandwidth",
onSelect: (ctx) => {
+206
View File
@@ -0,0 +1,206 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import type { HidPointerSample } from "../../g2/keyboard/hid";
import { keyboardService } from "../../g2/keyboard/service";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
import { clamp } from "../../util/numeric-util";
// How often to reconcile holding the pointer with window visibility.
const RECONCILE_INTERVAL_MS = 400;
/** A touchpad reports a hundred times a second; the page repaints at most this often. */
const RENDER_INTERVAL_MS = 33;
/** Cursor pixels per count the touchpad reports. */
const POINTER_GAIN = 1;
/** Left-click marks kept on the page. */
const MAX_MARKS = 12;
/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */
const CURSOR = [
"#",
"##",
"#.#",
"#..#",
"#...#",
"#....#",
"#.....#",
"#......#",
"#.......#",
"#........#",
"#.....#####",
"#..#..#",
"#.# #..#",
"## #..#",
"# #..#",
" #..#",
" ##",
];
/**
* Mouse cursor test: moves an arrow with the touchpad or mouse of the
* keyboard paired through the glasses (app/g2/keyboard/), and shows its
* buttons and wheel. The main UI has no cursor; this is where one is tried
* out.
*
* While the page is visible it takes the pointer, so the buttons and wheel
* show here instead of acting as select, back and scroll. A left click
* leaves a mark where the arrow is and a right click clears them. Leave
* with Esc, or a double tap on the ring or a temple.
*/
export class MouseTestLayer implements Layer {
private x = -1;
private y = -1;
private size = { width: 0, height: 0 };
private buttons = 0;
private wheel = 0;
private marks: Array<{ x: number; y: number }> = [];
private samples = 0;
private rate = 0;
private rateWindowStart = Date.now();
private removed = false;
private release: (() => void) | null = null;
private reconcileTimer: ReturnType<typeof setInterval> | null = null;
private renderTimer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly windowId: string,
private readonly requestRender: () => void,
) {
this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS);
this.reconcile();
}
/** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */
private reconcile(): void {
if (this.removed) return;
const visible = shell.isWindowVisible(this.windowId);
if (visible && !this.release) {
this.release = keyboardService.takePointer((sample) => this.onSample(sample));
} else if (!visible && this.release) {
this.release();
this.release = null;
this.buttons = 0;
}
// The rate reads zero once the pointer has been still for a second.
const now = Date.now();
if (now - this.rateWindowStart >= 1000) {
const rate = this.samples;
this.samples = 0;
this.rateWindowStart = now;
if (rate !== this.rate) {
this.rate = rate;
this.scheduleRender();
}
}
}
private onSample(sample: HidPointerSample): void {
if (this.removed) return;
this.samples++;
const { width, height } = this.size;
if (width > 0 && height > 0) {
if (this.x < 0) {
this.x = width / 2;
this.y = height / 2;
}
this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1);
this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1);
}
const pressed = sample.buttons & ~this.buttons;
this.buttons = sample.buttons;
this.wheel += sample.wheel;
if (pressed & 1 && this.x >= 0) {
this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) });
if (this.marks.length > MAX_MARKS) this.marks.shift();
}
if (pressed & 2) this.marks = [];
this.scheduleRender();
}
private scheduleRender(): void {
if (this.renderTimer !== null || this.removed) return;
this.renderTimer = setTimeout(() => {
this.renderTimer = null;
if (!this.removed) this.requestRender();
}, RENDER_INTERVAL_MS);
}
onRemoved(): void {
this.removed = true;
if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer);
this.reconcileTimer = null;
if (this.renderTimer !== null) clearTimeout(this.renderTimer);
this.renderTimer = null;
this.release?.();
this.release = null;
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height, padding } = ctx.stack.getBaseSize();
this.size = { width, height };
const image = new GrayImage(width, height, 0);
const left = padding.left;
const right = width - padding.right;
const top = padding.top;
const line = font.lineHeight + 4;
image.drawText(font, left, top, "Mouse cursor", 220);
const snapshot = keyboardService.snapshot();
const problem = snapshot.state !== "connected"
? "No keyboard is connected."
: !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null;
if (problem) {
image.drawText(font, left, top + 2 * line, problem, 190);
image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120);
return image;
}
const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`;
image.drawText(font, right - font.measureText(position), top, position, 150);
// Buttons: a box each, filled while held.
let x = left;
const boxY = top + line;
for (const [index, label] of ["L", "M", "R"].entries()) {
// HID numbers them left, right, middle; they sit left, middle, right.
const held = !!(this.buttons & [1, 4, 2][index]!);
const boxW = font.measureText(label) + 10;
if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200);
else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90);
image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170);
x += boxW + 4;
}
image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150);
const rate = `${this.rate}/s`;
image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110);
const hint = "Left click marks, right click clears. Esc or double tap: back";
image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90);
for (const mark of this.marks) {
image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140);
image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140);
}
if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y));
return image;
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
ctx.stack.pop();
}
}
}
function drawCursor(image: GrayImage, x: number, y: number): void {
for (const [row, pixels] of CURSOR.entries()) {
for (let col = 0; col < pixels.length; col++) {
const pixel = pixels[col];
if (pixel === " ") continue;
const px = x + col;
const py = y + row;
if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue;
image.setPixel(px, py, pixel === "#" ? 255 : 70);
}
}
}
+15 -1
View File
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
import { isFontFile } from "../../native/font-files";
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
import { isDecodableImageFile } from "../../native/image-files";
import { isMicRecordingFile } from "../../native/mic-recording";
import { isPlayableVideoFile } from "../../native/video-player";
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
openFontWindow: (title: string, path: string) => void;
/** Open a video file's player (settings panel first) as its own shell window. */
openVideoWindow: (title: string, path: string) => void;
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
openRecordingInTranscribe: (title: string, path: string) => void;
};
/**
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
const browser = new FileBrowserLayer({
isSupportedFile: (name) =>
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
// The browser handles double-click itself (up a level), so it is not
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
onLeave: () => shell.yieldFocusToSidebar(),
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
},
];
}
if (isMicRecordingFile(entry.name)) {
return [
{
label: "Open in Transcribe",
onSelect: (ctx) => {
ctx.stack.pop();
options.openRecordingInTranscribe(entry.name, entry.path);
},
},
];
}
if (isPlayableVideoFile(entry.name)) {
return [
{
+4
View File
@@ -14,6 +14,7 @@ import {
FILES_WINDOW_ID,
} from "./files-app";
import { createVideoPlayerWindow } from "./video-window";
import { openRecordingInTranscribe } from "../transcribe";
// Document windows are closeable and non-singleton; the serial keeps their
// windowIds unique.
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
},
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
openRecordingInTranscribe: (title, path) => {
openRecordingInTranscribe(ctx, path, title);
},
}),
),
// A document arriving via Android's Share intent opens as its own window.
+1
View File
@@ -305,6 +305,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
break;
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+1
View File
@@ -236,6 +236,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
case "navigation-sensors":
case "screen":
case "text-input":
case "key":
case "tool-call":
break;
}
+42 -18
View File
@@ -5,7 +5,9 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from
import { uiFontOverridePickerMenuItem } from "../../ui/font-picker";
import { type InputEvent } from "../../ui/gestures";
import { type Layer, type LayerContext } from "../../ui/layers";
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
import { drawSelectionHighlight, isMenuItemDisabled, menuHeadingItem, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
import { liveHeadPitchFooter } from "../../g2/head-angle";
import { headUpControlsAvailable } from "../../native/glasses-controls";
import { Menu } from "../../ui/menu-core";
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
@@ -20,7 +22,9 @@ import {
glanceLayout,
glanceLayoutSetting,
glanceShowLinesSetting,
glanceAngleSetting,
glanceShowOnHeadTiltSetting,
glanceTiltTimeoutSetting,
glanceShowOnLongPressSetting,
glanceSlotChoiceLabel,
glanceSlotSettings,
@@ -319,25 +323,45 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
}
/**
* Settings: layout, sleep gestures, duration, slot lines, widget font, and
* stereo depth, then a row per widget on the board for its own settings.
* Settings, in four sections: Layout (the slot grid), Trigger (the sleep
* gestures that show the board, and for how long), Display (slot lines,
* widget font, stereo depth) and Cards (a row per widget on the board, for
* its own settings; left out while the board is empty).
*/
function openGlanceSettings(ctx: LayerContext): void {
const items = (): MenuItem[] => [
// The layout decides which slots (and so which widgets) are on the board.
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
enumSettingMenuItem(glanceTapDurationSetting),
toggleSettingMenuItem(glanceShowOnLongPressSetting),
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
toggleSettingMenuItem(glanceShowLinesSetting),
uiFontOverridePickerMenuItem(glanceFont, {
title: "Glanceboard font",
description:
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
}),
enumSettingMenuItem(glanceDepthSetting),
...widgetSettingsMenuItems(),
];
const items = (): MenuItem[] => {
const cards = widgetSettingsMenuItems();
return [
menuHeadingItem("Layout"),
// The layout decides which slots (and so which widgets) are on the board.
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
menuHeadingItem("Trigger"),
enumSettingMenuItem(glanceTapDurationSetting),
toggleSettingMenuItem(glanceShowOnLongPressSetting),
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
enumSettingMenuItem(glanceTiltTimeoutSetting),
// Only Android can change the glasses' head-up setting (glasses-controls.ts).
// The controller's settings listener sends a new angle to the glasses; the
// picker shows the head's current angle to choose by.
...(headUpControlsAvailable()
? [enumSettingMenuItem(glanceAngleSetting, {
pickerFooter: liveHeadPitchFooter(() => shell.isWindowVisible(GLANCEBOARD_WINDOW_ID)),
})]
: []),
menuHeadingItem("Display"),
toggleSettingMenuItem(glanceShowLinesSetting),
uiFontOverridePickerMenuItem(glanceFont, {
title: "Glanceboard font",
description:
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
}),
enumSettingMenuItem(glanceDepthSetting),
...(cards.length ? [menuHeadingItem("Cards"), ...cards] : []),
];
};
const layer = new MenuLayer("Glanceboard settings", items(), PAGE_MENU_LAYOUT);
ctx.stack.push(layer);
}
+66 -8
View File
@@ -49,8 +49,9 @@ export function glanceLayout(): GlanceLayout {
/**
* Whether sleep-time gestures show the board at all. Off restores the plain
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
* wakes the regular UI), double-tap wakes.
* behaviour: a tap or hold while asleep does nothing, a head-tilt wakes the
* regular UI, double-tap wakes; and the head-up angle goes back to the glasses'
* own.
*/
export const glanceboardEnabledSetting = new ConfigSettingBoolean({
id: "glanceboard-enabled",
@@ -85,9 +86,10 @@ function formatTapDuration(value: GlanceTapDuration): string {
}
/**
* How long a tap (or head-tilt) keeps the board up; each further tap restarts
* it. "off" means a single tap does not show the board at all; a head-tilt
* then uses the default duration.
* How long a tap keeps the board up; each further tap restarts it. "off"
* means a single tap does not show the board at all. A head-tilt showing
* lasts until the head comes back down, falling back to this duration (the
* default one when "off") where the head-down is unlikely to arrive.
*/
export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>({
id: "glanceboard-tap-duration",
@@ -97,7 +99,7 @@ export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>
values: TAP_DURATION_VALUES,
formatValue: formatTapDuration,
description:
"How long a single tap (or a head-tilt) keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
"How long a single tap keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
});
/** Whether a single tap while asleep shows the board. */
@@ -119,15 +121,71 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
});
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
/** The head-tilt wake shows the board (until the head comes back down) instead of the regular UI. */
export const glanceShowOnHeadTiltSetting = new ConfigSettingBoolean({
id: "glanceboard-show-on-head-tilt",
label: "Show on head tilt",
storageKey: "glanceboard.showOnHeadTilt",
defaultValue: true,
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI.",
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI. It stays up until you look back down.",
});
export type GlanceTiltTimeout = "60s" | "30s" | "15s" | "10s" | "never";
/**
* How long a head-tilt showing may last if the head-down never comes (the
* board normally hides when the head comes back down). "never" removes the
* limit; a double-tap still dismisses the board.
*/
export const glanceTiltTimeoutSetting = new ConfigSettingEnum<GlanceTiltTimeout>({
id: "glanceboard-head-tilt-timeout",
label: "Head tilt timeout",
storageKey: "glanceboard.headTiltTimeout",
defaultValue: "60s",
values: ["60s", "30s", "15s", "10s", "never"],
formatValue: (value) => (value === "never" ? "Never" : value.replace("s", " s")),
description:
"The longest a head-tilt keeps the Glanceboard up when looking back down is not detected. Never: it stays until you look down or double-tap. " +
"When looking down is unlikely to register (a low Head tilt angle, or the display was suspended), the tap duration applies if shorter.",
});
/** The Head tilt timeout in ms, or null for none. */
export function glanceTiltTimeoutMs(): number | null {
const value = glanceTiltTimeoutSetting.get();
return value === "never" ? null : Number.parseInt(value, 10) * 1000;
}
const GLANCE_ANGLE_VALUES = ["glasses", "5", "10", "15", "20", "25", "30", "35", "40", "45", "50", "55", "60"] as const;
export type GlanceAngle = (typeof GLANCE_ANGLE_VALUES)[number];
/**
* How far the head must tilt up for a head-tilt wake, as the glasses' own
* pitch reading (degrees). "glasses" keeps the glasses' own setting (from the
* Even app), writing back their original values if Faceclaw changed them. A
* number is written to the glasses' head-up setting on every connection while
* the Glanceboard is enabled (head-angle.ts). Android only.
*/
export const glanceAngleSetting = new ConfigSettingEnum<GlanceAngle>({
id: "glanceboard-head-tilt-angle",
label: "Head tilt angle",
storageKey: "glanceboard.headTiltAngle",
defaultValue: "glasses",
values: GLANCE_ANGLE_VALUES,
formatValue: (value) => (value === "glasses" ? "Glasses default" : `${value}°`),
description:
"How far you tilt your head up to wake the display; the list shows your head's angle now. Glasses default uses the glasses' own setting from the Even app. " +
"The board hides when your head drops 20° below this angle; under 25°, looking straight ahead may not count, so it hides after the tap duration instead.",
});
/**
* The trigger Faceclaw should maintain, in degrees: the Head tilt angle while
* the Glanceboard is enabled, else 0 (the glasses' own setting).
*/
export function glanceAngleTriggerDegrees(): number {
const value = glanceAngleSetting.get();
return value === "glasses" || !glanceboardEnabledSetting.get() ? 0 : Number(value);
}
const DEPTH_VALUES = ["-64", "-48", "-32", "-16", "0", "16", "32", "48", "64"] as const;
export type GlanceDepth = (typeof DEPTH_VALUES)[number];
+2
View File
@@ -9,6 +9,7 @@ import {
glanceShowOnLongPressSetting,
glanceShowOnTap,
glanceTapTimeoutMs,
glanceTiltTimeoutMs,
} from "./glanceboard-settings";
const glanceboardApp: AppDefinition = {
@@ -24,6 +25,7 @@ const glanceboardApp: AppDefinition = {
isEnabled: () => glanceboardEnabledSetting.get(),
showOnTap: glanceShowOnTap,
tapTimeoutMs: glanceTapTimeoutMs,
tiltTimeoutMs: glanceTiltTimeoutMs,
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
depth: glanceDepth,
+15 -5
View File
@@ -50,16 +50,26 @@ export const musicShowAlbumArtSetting = new ConfigSettingBoolean({
"Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.",
});
export const musicShowIdleNoteSetting = new ConfigSettingBoolean({
id: "glanceboard-music-show-idle-note",
label: "Show note when idle",
storageKey: "glanceboard.music.showIdleNote",
defaultValue: true,
description: "Show a dim music note on the card while nothing is playing. When off, the card is left blank.",
});
/** The music card's submenu in Glanceboard > Settings. */
export function musicSettingsMenuItems(): MenuItem[] {
return [toggleSettingMenuItem(musicShowAlbumArtSetting)];
return [toggleSettingMenuItem(musicShowAlbumArtSetting), toggleSettingMenuItem(musicShowIdleNoteSetting)];
}
/**
* Now playing: album art (unless turned off), title/artist/album and a
* progress bar for the active media session (Apple Music on iOS), the Music
* app's header at card size. The media bridge runs for the whole session; the
* widget subscribes and ticks once a second only while something is playing.
* app's header at card size. With nothing playing the card holds a dim note,
* or nothing with that turned off. The media bridge runs for the whole
* session; the widget subscribes and ticks once a second only while
* something is playing.
*/
export class MusicWidget implements GlanceWidget {
private media: MediaControllerState = mediaControllerBridge.snapshot();
@@ -108,8 +118,8 @@ export class MusicWidget implements GlanceWidget {
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
return;
}
// Nothing playing: a dim note, centred, and nothing else.
const note = idleNoteImage();
// Nothing playing: a dim note, centred, and nothing else; or a blank card.
const note = musicShowIdleNoteSetting.get() ? idleNoteImage() : null;
if (note) image.drawImage(note, ((image.width - note.width) / 2) | 0, ((image.height - note.height) / 2) | 0);
return;
}
+21
View File
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
}
/**
* The configuration a capture of the temples' own microphones arms them
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
* of each. The front end and direction detection follow Microphone setup
* (the Microphones app), except that both of a temple's microphones only
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
* what the firmware streams and transcription takes.
*/
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
const base = loadMicConfig();
return {
...base,
source: mics === 4 ? "codec" : base.source,
channelMask: mics === 4 ? 0b11 : 0b01,
codec: "raw",
format: "pcm16",
sampleRateHz: 16_000,
armHardware: true,
};
}
class MicArrayController {
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
private statuses: MicArrayStatuses = { left: null, right: null };
@@ -272,6 +272,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+3
View File
@@ -370,6 +370,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (editing) {
// Voice input as an alternative to the phone keyboard.
+1
View File
@@ -433,6 +433,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+3
View File
@@ -227,6 +227,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (editingSetting) {
// Voice input as an alternative to the phone keyboard.
+3
View File
@@ -424,6 +424,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (window && window.windowId === message.windowId) handleTextInput(message.text.trim());
break;
+89 -7
View File
@@ -1,13 +1,21 @@
import { getStringSetting, setStringSetting } from "../../native/settings-store";
/**
* Terminal-app connection registry: the g2mirror hosts the app knows about,
* stored as JSON under one settings key so every isolate sees the same list
* and change notifications ride the ordinary settings-store channel.
* Terminal-app connection registry: the hosts the app knows about, stored as
* JSON under one settings key so every isolate sees the same list and change
* notifications ride the ordinary settings-store channel.
*
* A connection is entered as a g2mirror:// connection string (the same form
* g2mirror-view accepts): g2mirror://<token>@<host>[:port] for plain
* websockets (default port 8737), g2mirrors://... for TLS (default 443).
* A connection is entered as a connection string:
* - g2mirror://<token>@<host>[:port]: a g2mirror server reached directly
* (the same form g2mirror-view accepts) over a plain websocket (default
* port 8737); g2mirrors://... for TLS (default 443).
* - ssh://<user>@<host>[:port]: tmux on an SSH host (default port 22).
* - ssh://<user>@<host>[:port]?g2mirror=<token>[@<host>[:port]]: a g2mirror
* server tunnelled over SSH; the address after the token is where the
* server listens as seen from the SSH host (default 127.0.0.1:8737).
* SSH connections pin the host's key (hostKeys, trusted on first connection
* after the user checks its fingerprint); their passwords live in the
* platform keystore (app/native/ssh), not here.
*/
export const TERMINAL_CONNECTIONS_KEY = "terminal.connections";
@@ -25,6 +33,8 @@ export type TerminalConnection = {
serverName: string | null;
/** Auto-connect while the Terminal app is open. Toggled by Connect/Disconnect. */
enabled: boolean;
/** SSH connections: trusted host keys (SSH wire format, base64). */
hostKeys: string[];
};
export type ParsedConnectionString = {
@@ -54,8 +64,79 @@ export function parseConnectionString(url: string): ParsedConnectionString | nul
return { secure, host, port, authToken };
}
/** The host[:port] part of a connection string; "" if it doesn't parse. */
export type ParsedSshConnection = {
user: string;
host: string;
port: number;
} & (
| { mode: "tmux" }
| {
mode: "g2mirror";
authToken: string;
/** Where g2mirror-server listens, from the SSH host's point of view. */
targetHost: string;
targetPort: number;
}
);
function decodeComponent(text: string): string {
try {
return decodeURIComponent(text);
} catch {
return text;
}
}
/** Parse an ssh://user@host[:port][?g2mirror=token[@host[:port]]] string; null if malformed. */
export function parseSshConnectionString(url: string): ParsedSshConnection | null {
const match = /^ssh:\/\/([^@\s/?]+)@(\[[^\]\s]+\]|[^:/@\s?]+)(?::(\d+))?\/?(?:\?(.*))?$/.exec(url.trim());
if (!match) return null;
const user = decodeComponent(match[1]!);
const host = match[2]!.replace(/^\[(.*)\]$/, "$1");
const port = match[3] ? parseInt(match[3], 10) : 22;
if (!user || !(port >= 1 && port <= 65535)) return null;
const params = new Map<string, string>();
for (const part of (match[4] ?? "").split("&").filter(Boolean)) {
const equals = part.indexOf("=");
if (equals < 0) return null;
params.set(part.slice(0, equals), decodeComponent(part.slice(equals + 1)));
}
for (const key of params.keys()) {
if (key !== "g2mirror") return null;
}
const g2mirror = params.get("g2mirror");
if (g2mirror === undefined) return { user, host, port, mode: "tmux" };
// token[@host[:port]], split at the last "@" so the token may contain one.
const at = g2mirror.lastIndexOf("@");
const authToken = at < 0 ? g2mirror : g2mirror.slice(0, at);
const target = at < 0 ? "" : g2mirror.slice(at + 1);
const targetMatch = /^(\[[^\]\s]+\]|[^:\s]*)(?::(\d+))?$/.exec(target);
if (!authToken || !targetMatch) return null;
const targetHost = targetMatch[1] || "127.0.0.1";
const targetPort = targetMatch[2] ? parseInt(targetMatch[2], 10) : 8737;
if (!(targetPort >= 1 && targetPort <= 65535)) return null;
return { user, host, port, mode: "g2mirror", authToken, targetHost, targetPort };
}
export type ParsedAnyConnection =
| ({ kind: "g2mirror" } & ParsedConnectionString)
| ({ kind: "ssh" } & ParsedSshConnection);
/** Parse any supported connection string; null if it's none of them. */
export function parseAnyConnectionString(url: string): ParsedAnyConnection | null {
const direct = parseConnectionString(url);
if (direct) return { kind: "g2mirror", ...direct };
const ssh = parseSshConnectionString(url);
return ssh ? { kind: "ssh", ...ssh } : null;
}
/** The [user@]host[:port] part of a connection string (never a token); "" if it doesn't parse. */
export function connectionStringHostLabel(url: string): string {
const ssh = parseSshConnectionString(url);
if (ssh) {
const host = ssh.host.includes(":") ? `[${ssh.host}]` : ssh.host;
return `${ssh.user}@${host}${ssh.port === 22 ? "" : `:${ssh.port}`}`;
}
const parsed = parseConnectionString(url);
if (!parsed) return "";
const defaultPort = parsed.secure ? 443 : 8737;
@@ -79,6 +160,7 @@ export function loadConnections(): TerminalConnection[] {
url: String(item?.url ?? ""),
serverName: typeof item?.serverName === "string" && item.serverName ? item.serverName : null,
enabled: item?.enabled !== false,
hostKeys: Array.isArray(item?.hostKeys) ? item.hostKeys.filter((key: unknown) => typeof key === "string") : [],
}))
.filter((connection) => connection.id.length > 0 && connection.url.length > 0);
} catch {
+2
View File
@@ -14,6 +14,8 @@ const terminalApp: AppDefinition = {
// Session windows stay independent: launching reopens the hub even
// while terminal sessions are open.
matchExistingBy: "windowId",
// The hub window can show a session too, so it takes keys like one.
acceptsKeys: true,
}),
};
+500 -36
View File
@@ -16,11 +16,17 @@
* Each view has its own websocket connection (the protocol allows one attached
* session per connection) and its own xterm emulator.
*
* Connections are g2mirror://<token>@<host>[:port] strings (g2mirrors:// for
* TLS), stored as JSON in the settings store (see connections.ts). Each
* enabled connection gets a control websocket that backs the session list and
* carries unsolicited bell/title notifications; control connections live as
* long as the worker so bells keep flowing even if the hub window is closed.
* Connections are connection strings stored as JSON in the settings store
* (see connections.ts): g2mirror://<token>@<host>[:port] for a g2mirror
* server (g2mirrors:// for TLS), ssh://<user>@<host> for tmux over SSH
* (TmuxClient), or ssh://<user>@<host>?g2mirror=<token> for a g2mirror server
* tunnelled over SSH. Each enabled connection gets a control client that
* backs the session list and carries bell/title/activity notifications;
* control connections live as long as the worker so bells keep flowing even
* if the hub window is closed. SSH connections that need the user (an
* unverified or changed host key, a password, an unauthorized key) record
* the failure on the control (sshIssue) and offer a page that resolves it
* instead of retrying.
* The host's server_name (from the init reply) is cached per connection and
* used as its display name, falling back to the connection string's
* host[:port] before the first successful handshake.
@@ -55,7 +61,24 @@ import {
import * as frameTimings from "../../native/frame-timings";
import { getActiveDisplay } from "../../native/active-display";
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
import type { KeyEvent } from "../../g2/keyboard/keys";
import { encodeTerminalKey, terminalScrollbackKey } from "./terminal-keys";
import { G2MirrorClient, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
import type { TerminalClient } from "./terminal-client";
import { TmuxClient, type OpenLineChannel } from "./tmux-client";
import {
copyToPhoneClipboard,
describeSshFailure,
getSshSecret,
openSshChannel,
removeSshSecret,
setSshSecret,
sshClientKey,
type SshEndpoint,
type SshFailure,
type SshHostKey,
} from "../../native/ssh";
import { wrapText } from "../../graphics/textwrap";
import { onSettingsStoreChanged } from "../../native/settings-store";
import { clamp } from "../../util/numeric-util";
import {
@@ -64,12 +87,22 @@ import {
terminalDisplayLayoutSetting,
terminalLaunchPresetsSetting,
terminalNewConnectionSetting,
terminalSshPasswordSetting,
terminalWakeOnBellSetting,
textSettingMenuItem,
toggleSettingMenuItem,
} from "../../ui/dashboard-settings";
import { terminalFontPickerMenuItem } from "../../ui/font-picker";
import { connectionDisplayName, loadConnections, parseConnectionString, saveConnections, TERMINAL_CONNECTIONS_KEY, updateConnection, type TerminalConnection } from "./connections";
import {
connectionDisplayName,
connectionStringHostLabel,
loadConnections,
parseAnyConnectionString,
saveConnections,
TERMINAL_CONNECTIONS_KEY,
updateConnection,
type TerminalConnection,
} from "./connections";
import { TerminalEmulator } from "./terminal-emulator";
import { drawSubmenuIndicator, submenuItem, type MenuItem } from "../../ui/menu";
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
@@ -112,6 +145,10 @@ const RECONNECT_MAX_DELAY_MS = 60_000;
const NEW_CONNECTION_DRAFT_KEY = "terminal.newConnectionDraft";
// Storage key of terminalLaunchPresetsSetting, edited from the window menu.
const LAUNCH_PRESETS_KEY = "terminal.launchPresets";
// Storage key of terminalSshPasswordSetting (the password page's draft).
const SSH_PASSWORD_DRAFT_KEY = "terminal.sshPasswordDraft";
// Comment on this device's SSH public key in authorized_keys.
const SSH_KEY_COMMENT = "faceclaw";
type BaseWindow = {
windowId: string;
@@ -148,10 +185,24 @@ type BaseWindow = {
/**
* What the hub window is showing: the session list, the Manage Connections
* section, or the Add-connection screen (type the g2mirror:// string on the
* phone, click to save).
* section, the Add-connection screen (type the connection string on the
* phone, click to save), or a page of text with a click action (verify a
* host key, enter a password, show this device's SSH key).
*/
type HubMode = "sessions" | "connections" | "add";
type HubMode = "sessions" | "connections" | "add" | "page";
type HubPage = {
title: string;
/** Body text, wrapped to the window; re-read on every paint. */
body: () => string;
/** Bottom hint line. */
hint: string;
onClick?: () => void;
/** Double-click. */
onBack: () => void;
/** Runs whenever the page goes away (back, click, window closed). */
onLeave?: () => void;
};
type HubWindow = BaseWindow & {
kind: "hub";
@@ -168,6 +219,8 @@ type HubWindow = BaseWindow & {
sessionOrder: string[];
/** Parse-failure message shown on the Add-connection screen. */
addError: string;
/** The page shown in mode "page". */
page: HubPage | null;
};
type ViewWindow = BaseWindow & {
@@ -186,7 +239,7 @@ type ViewWindow = BaseWindow & {
label: string;
/** Sidebar-icon character (">3"); "" if every glyph was taken at open time. */
glyph: string;
client: G2MirrorClient;
client: TerminalClient;
emulator: TerminalEmulator;
receivedData: boolean;
attachRequested: boolean;
@@ -220,8 +273,8 @@ type PendingView = {
socket: string;
label: string;
glyph: string;
/** Client options captured when the view was requested from a live control. */
options: G2MirrorClientOptions;
/** How to make the view's client, captured when it was requested from a live control. */
options: ClientRecipe;
};
const windows = new Map<string, TerminalWindow>();
@@ -302,7 +355,13 @@ let screenOn = true;
*/
type ControlConnection = {
config: TerminalConnection;
client: G2MirrorClient | null;
client: TerminalClient | null;
/**
* An SSH failure only the user can fix (host key to verify, password or
* key needed). Shown on the host's heading with a row that opens the fix;
* reconnects wait for it. Cleared on a successful connection.
*/
sshIssue: SshFailure | null;
state: G2MirrorState | null;
unsubscribers: Array<() => void>;
reconnectTimer: ReturnType<typeof setTimeout> | null;
@@ -435,6 +494,13 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
break;
}
case "key": {
const window = windows.get(message.windowId);
if (!window) break;
window.focused = message.focused;
handleKey(window, message.event as KeyEvent);
break;
}
case "render": {
const window = windows.get(message.windowId);
if (!window) break;
@@ -504,7 +570,8 @@ onSettingsStoreChanged((key) => {
reportIdle();
return;
case NEW_CONNECTION_DRAFT_KEY:
// Live keystrokes from the phone editor; repaint the add screen.
case SSH_PASSWORD_DRAFT_KEY:
// Live keystrokes from the phone editor; repaint the add/password screen.
renderHubWindows();
return;
case "terminal.autoReconnect":
@@ -578,6 +645,7 @@ function openWindow(windowId: string, surfaceId: string, title: string, viewport
list: null,
sessionOrder: [],
addError: "",
page: null,
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
@@ -610,6 +678,10 @@ function closeWindow(windowId: string): void {
// Window closed mid-add: shut the phone editor down.
endAddConnection(window);
}
if (window.kind === "hub" && window.mode === "page") {
// Likewise for a page's phone editor (the password draft).
closeHubPage(window, "sessions");
}
// Likewise for a setting being edited from the window menu.
window.menu?.close();
windows.delete(windowId);
@@ -627,17 +699,120 @@ function closeWindow(windowId: string): void {
}
}
/** Client options for a connection string; null if the string doesn't parse. */
function clientOptionsFor(connection: TerminalConnection): G2MirrorClientOptions | null {
const parsed = parseConnectionString(connection.url);
/**
* How to make clients for a connection: "g2mirror" dials a g2mirror server
* directly, "ssh-g2mirror" reaches one through an SSH tunnel, "tmux" runs
* tmux control mode over SSH.
*/
type ClientRecipe = {
kind: "g2mirror" | "ssh-g2mirror" | "tmux";
/**
* `role` "control" watches bells (tmux keeps the bell bookkeeping for that
* client only). `onSshFailure` hears why an SSH channel failed.
*/
create(cols: number, rows: number, role: "control" | "view", onSshFailure?: (failure: SshFailure) => void): TerminalClient;
};
/** Client recipe for a connection string; null if the string doesn't parse. */
function clientOptionsFor(connection: TerminalConnection): ClientRecipe | null {
const parsed = parseAnyConnectionString(connection.url);
if (!parsed) return null;
if (parsed.kind === "g2mirror") {
const { secure, host, port, authToken } = parsed;
return {
kind: "g2mirror",
create: (cols, rows) => new G2MirrorClient({ secure, host, port, authToken, deviceName: DEVICE_NAME, cols, rows }),
};
}
const connectionId = connection.id;
const label = connectionStringHostLabel(connection.url);
// Read at every channel open, so a newly trusted key or saved password
// applies to the next (re)connect, views included.
const endpoint = (): SshEndpoint => ({
host: parsed.host,
port: parsed.port,
user: parsed.user,
hostKeys: loadConnections().find((candidate) => candidate.id === connectionId)?.hostKeys ?? [],
password: getSshSecret(sshPasswordSecretId(connectionId)),
useKey: true,
});
if (parsed.mode === "g2mirror") {
const { authToken, targetHost, targetPort } = parsed;
const urlHost = targetHost.includes(":") && !targetHost.startsWith("[") ? `[${targetHost}]` : targetHost;
const url = `ws://${urlHost}:${targetPort}/`;
return {
kind: "ssh-g2mirror",
create: (cols, rows, _role, onSshFailure) =>
new G2MirrorClient({
secure: false,
host: targetHost,
port: targetPort,
authToken,
deviceName: DEVICE_NAME,
cols,
rows,
transport: {
label: `${label} over SSH`,
open: (listener) => {
const channel = openSshChannel(endpoint(), { type: "websocket", url }, {
onOpen: () => listener.onOpen(),
onText: (text) => listener.onTextMessage(text),
onClosed: (info) => listener.onClosed(info.code ?? 1000, info.reason ?? ""),
onFailure: (failure) => {
onSshFailure?.(failure);
listener.onFailure(describeSshFailure(failure));
},
});
return { sendText: (text) => channel.send(text), sendBinary: () => false, close: () => channel.close() };
},
},
}),
};
}
return {
...parsed,
deviceName: DEVICE_NAME,
...viewGrid(),
kind: "tmux",
create: (cols, rows, role, onSshFailure) => {
const openChannel: OpenLineChannel = (command, listener) =>
openSshChannel(endpoint(), { type: "exec", command }, {
onOpen: () => listener.onOpen(),
onText: (line) => listener.onLine(line),
onStderr: (line) => listener.onStderr(line),
onClosed: (info) => listener.onClosed(info.exitStatus ?? null),
onFailure: (failure) => {
onSshFailure?.(failure);
listener.onFailure(describeSshFailure(failure));
},
});
return new TmuxClient({ openChannel, label, cols, rows, stateKey: role === "control" ? connectionId : "" });
},
};
}
function sshPasswordSecretId(connectionId: string): string {
return `terminal.password.${connectionId}`;
}
/** Failures that retrying can't fix: the user has to verify a key or supply credentials. */
function sshFailureNeedsUser(failure: SshFailure): boolean {
return failure.kind === "hostkey-unknown" || failure.kind === "hostkey-mismatch" || failure.kind === "auth";
}
/**
* A control's SSH channel failed. If the user has to act, remember why, and
* if they're looking at the hub's lists, open the page that fixes it.
*/
function noteSshFailure(control: ControlConnection, failure: SshFailure): void {
if (!sshFailureNeedsUser(failure) || controls.get(control.config.id) !== control || !control.client) return;
control.sshIssue = failure;
renderHubWindows();
for (const window of windows.values()) {
if (window.kind === "hub" && window.foreground && (window.mode === "sessions" || window.mode === "connections")) {
openSshIssuePage(window, control);
break;
}
}
}
/**
* Reconcile the live control clients with the stored connection list: start
* enabled connections, stop disabled or removed ones, restart ones whose
@@ -663,6 +838,7 @@ function syncControlsFromSettings(): void {
control = {
config: connection,
client: null,
sshIssue: null,
state: null,
unsubscribers: [],
reconnectTimer: null,
@@ -684,12 +860,13 @@ function syncControlsFromSettings(): void {
function startControl(control: ControlConnection): void {
stopControl(control);
const options = clientOptionsFor(control.config);
if (!options) {
const recipe = clientOptionsFor(control.config);
if (!recipe) {
control.state = null;
return;
}
const client = new G2MirrorClient(options);
const grid = viewGrid();
const client = recipe.create(grid.cols, grid.rows, "control", (failure) => noteSshFailure(control, failure));
const connectionId = control.config.id;
control.client = client;
control.state = client.state();
@@ -703,6 +880,7 @@ function startControl(control: ControlConnection): void {
control.state = state;
if (state.phase === "connected") {
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
control.sshIssue = null;
maybeCacheServerName(control, state.serverName);
}
if (state.phase === "failed") scheduleControlReconnect(control);
@@ -777,6 +955,8 @@ function scheduleControlReconnect(control: ControlConnection): void {
if (!terminalAutoReconnectSetting.get()) return;
if (windows.size === 0 && !hasTerminalBackgroundWork()) return;
if (!control.config.enabled) return;
// Waiting on the user (host key, password) or on the host (no usable tmux).
if (control.sshIssue || control.state?.retryable === false) return;
const delayMs = control.reconnectDelayMs;
control.reconnectDelayMs = Math.min(control.reconnectDelayMs * 2, RECONNECT_MAX_DELAY_MS);
control.reconnectTimer = setTimeout(() => {
@@ -943,7 +1123,7 @@ function createViewWindow(
const cell = getTerminalFontConfig();
const gridCols = Math.floor(viewport.width / cell.cellWidth);
const gridRows = Math.floor(viewport.height / cell.cellHeight);
const client = new G2MirrorClient({ ...view.options, cols: gridCols, rows: gridRows });
const client = view.options.create(gridCols, gridRows, "view");
const window: ViewWindow = {
kind: "view",
hub: null,
@@ -1238,6 +1418,11 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
renderAndSubmit(window, frameId);
return;
}
if (window.kind === "hub" && window.mode === "page") {
window.page?.onBack();
renderAndSubmit(window, frameId);
return;
}
if (window.kind === "hub" && window.mode === "connections") {
setHubMode(window, "sessions");
renderAndSubmit(window, frameId);
@@ -1260,6 +1445,35 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
frameTimings.finishFrame(frameId, "discarded: terminal view ignored input");
}
/**
* A key from a hardware keyboard (the glasses' keyboard bridge) that no
* keybinding claimed. A session with nothing over it gets the key as its
* terminal would: Escape and the arrows belong to the program, and the way
* out is Escape with Shift, Ctrl or Alt, which never arrives here (it is a
* built-in keybinding for "back"). Anywhere else (the hub's lists, an open
* menu) the key is handed back, to act as navigation.
*/
function handleKey(window: TerminalWindow, event: KeyEvent): void {
if (window.kind === "view" && !window.menu?.isOpen()) {
activeViewId = window.windowId;
const page = terminalScrollbackKey(event);
if (page !== null) {
handleViewScroll(window, page, 0);
return;
}
const bytes = encodeTerminalKey(event, window.emulator.applicationCursorKeys());
if (bytes === null) return;
// Typing follows the live screen, as in any terminal emulator.
if (window.scrollTop !== null) {
window.scrollTop = null;
scheduleRender(window);
}
window.client.sendInput(bytes);
return;
}
post({ type: "key-unhandled", windowId: window.windowId, event });
}
/** Top visible absolute line index when following the live bottom. */
function followTop(window: ViewWindow): number {
return window.historyNext + window.emulator.bufferLength() - window.gridRows;
@@ -1386,6 +1600,154 @@ function setHubMode(window: HubWindow, mode: HubMode): void {
window.addError = "";
}
function openHubPage(window: HubWindow, page: HubPage): void {
if (window.mode === "add") endAddConnection(window);
if (window.mode === "page") window.page?.onLeave?.();
window.page = page;
window.mode = "page";
scheduleRender(window);
}
/** Leave the current page (if any) for `mode`. */
function closeHubPage(window: HubWindow, mode: HubMode): void {
const page = window.page;
window.page = null;
page?.onLeave?.();
setHubMode(window, mode);
scheduleRender(window);
}
/** The list a page returns to: the one it was opened from. */
function hubListMode(window: HubWindow): HubMode {
return window.mode === "connections" ? "connections" : "sessions";
}
function openSshIssuePage(window: HubWindow, control: ControlConnection): void {
const issue = control.sshIssue;
if (!issue) return;
if (issue.kind === "hostkey-unknown" && issue.hostKey) {
openHostKeyPage(window, control, issue.hostKey);
} else if (issue.kind === "hostkey-mismatch" && issue.hostKey) {
openHostKeyChangedPage(window, control, issue.hostKey);
} else if (issue.kind === "auth" && issue.passwordAllowed) {
openPasswordPage(window, control, Boolean(issue.passwordTried));
} else if (issue.kind === "auth") {
openClientKeyPage(window, `${connectionDisplayName(control.config)} didn't accept this device's key (and doesn't take passwords).\n`);
}
}
function hostKeyTypeName(type: string): string {
if (type === "ssh-ed25519") return "ED25519";
if (type.startsWith("ecdsa-")) return "ECDSA";
if (type === "ssh-rsa") return "RSA";
return type;
}
function hostKeyFile(type: string): string {
if (type === "ssh-ed25519") return "/etc/ssh/ssh_host_ed25519_key.pub";
if (type.startsWith("ecdsa-")) return "/etc/ssh/ssh_host_ecdsa_key.pub";
return "/etc/ssh/ssh_host_rsa_key.pub";
}
/** First connection to an SSH host: show the key's fingerprint; click trusts it. */
function openHostKeyPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
const returnMode = hubListMode(window);
openHubPage(window, {
title: "Verify host key",
body: () =>
`First connection to ${connectionStringHostLabel(control.config.url)}. Its ${hostKeyTypeName(hostKey.type)} host key fingerprint is:\n` +
`${hostKey.fingerprint}\n\n` +
`To check it, run this on the host and compare:\nssh-keygen -lf ${hostKeyFile(hostKey.type)}`,
hint: `click trust and connect ${GESTURE_DOUBLE_CLICK} cancel`,
onClick: () => {
const hostKeys = [...control.config.hostKeys.filter((key) => key !== hostKey.key), hostKey.key];
control.config.hostKeys = hostKeys;
updateConnection(control.config.id, { hostKeys });
control.sshIssue = null;
closeHubPage(window, returnMode);
connectControl(control);
},
onBack: () => closeHubPage(window, returnMode),
});
}
function openHostKeyChangedPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
const returnMode = hubListMode(window);
openHubPage(window, {
title: "HOST KEY CHANGED",
body: () =>
`${connectionStringHostLabel(control.config.url)} presented a different host key than the one trusted before: ` +
`${hostKeyTypeName(hostKey.type)} ${hostKey.fingerprint}\n\n` +
"Someone may be intercepting the connection, so Faceclaw won't connect. If the host was reinstalled or its keys " +
'were regenerated, use "Forget host key" on this connection in Manage Connections, then verify the new fingerprint.',
hint: `${GESTURE_DOUBLE_CLICK} back`,
onBack: () => closeHubPage(window, returnMode),
});
}
/** Ask for the host's password on the phone (masked editor); click saves it and connects. */
function openPasswordPage(window: HubWindow, control: ControlConnection, rejected: boolean): void {
const returnMode = hubListMode(window);
terminalSshPasswordSetting.set("");
post({ type: "start-text-setting-edit", settingId: terminalSshPasswordSetting.id });
openHubPage(window, {
title: "SSH password",
body: () => {
const typed = terminalSshPasswordSetting.get();
return (
(rejected ? "The host refused the saved password.\n" : "") +
`Type the password for ${connectionStringHostLabel(control.config.url)} in the phone app, then click to connect. ` +
"It is saved encrypted on this phone.\n\n" +
(typed ? "*".repeat(Math.min(typed.length, 32)) : "(nothing typed yet)")
);
},
hint: `click connect ${GESTURE_DOUBLE_CLICK} cancel`,
onClick: () => {
const password = terminalSshPasswordSetting.get();
if (!password) return;
setSshSecret(sshPasswordSecretId(control.config.id), password);
control.sshIssue = null;
closeHubPage(window, returnMode);
connectControl(control);
},
onBack: () => closeHubPage(window, returnMode),
onLeave: () => {
terminalSshPasswordSetting.set("");
post({ type: "end-text-setting-edit" });
},
});
}
/** This device's SSH public key (the private half never leaves the keystore); click copies it. */
function openClientKeyPage(window: HubWindow, intro = ""): void {
const returnMode = hubListMode(window);
let key: { authorizedKey: string; fingerprint: string } | null = null;
let error = "";
try {
key = sshClientKey(SSH_KEY_COMMENT);
} catch (caught) {
error = String((caught as Error)?.message ?? caught);
}
let copied = false;
openHubPage(window, {
title: "This device's SSH key",
body: () =>
key
? `${intro}Add this line to ~/.ssh/authorized_keys on each host to log in with this phone's key:\n` +
`${key.authorizedKey}\n${key.fingerprint}` +
(copied ? "\n\nCopied to the phone's clipboard." : "")
: `Couldn't create this device's key: ${error}`,
hint: key ? `click copy to phone clipboard ${GESTURE_DOUBLE_CLICK} back` : `${GESTURE_DOUBLE_CLICK} back`,
onClick: key
? () => {
copyToPhoneClipboard("SSH public key", key!.authorizedKey);
copied = true;
}
: undefined,
onBack: () => closeHubPage(window, returnMode),
});
}
/**
* One control connection's sessions in this hub window's display order:
* most-recently-updated first as of when the window became visible, with
@@ -1441,6 +1803,7 @@ function hubSessionItems(window: HubWindow): HubItem[] {
];
const multiHost = controls.size > 1;
const canLaunch = launchPresetNames().length > 0;
const isTmux = (control: ControlConnection) => clientOptionsFor(control.config)?.kind === "tmux";
// Every configured host is listed under its own heading, in stored order,
// whatever its connection state: a connected host lists its sessions, any
// other shows its state on the heading plus a Connect row (unless it is
@@ -1450,10 +1813,17 @@ function hubSessionItems(window: HubWindow): HubItem[] {
const name = connectionDisplayName(control.config);
if (!isControlConnected(control)) {
items.push({ label: `${name} (${controlStatusWord(control)})`, heading: true });
if (control.sshIssue && control.config.enabled) {
items.push({ label: sshIssueAction(control.sshIssue), onSelect: () => openSshIssuePage(window, control) });
continue;
}
const connecting = control.config.enabled && control.state?.phase === "connecting";
if (!connecting && clientOptionsFor(control.config)) {
// A failure retrying can't fix (tmux missing or too old) says why;
// selecting it retries once the host is fixed.
const unfixable = control.config.enabled && control.state?.retryable === false;
items.push({
label: control.reconnectTimer ? "Connect now" : "Connect",
label: unfixable ? control.state!.status : control.reconnectTimer ? "Connect now" : "Connect",
onSelect: () => connectControl(control),
});
}
@@ -1462,7 +1832,8 @@ function hubSessionItems(window: HubWindow): HubItem[] {
items.push({
label: name,
heading: true,
onSelect: canLaunch ? () => launchOnHost(window, control) : undefined,
// tmux hosts can always start a shell, presets or not.
onSelect: canLaunch || isTmux(control) ? () => launchOnHost(window, control) : undefined,
});
const sessions = orderedSessions(window, control);
for (const session of sessions) {
@@ -1478,7 +1849,11 @@ function hubSessionItems(window: HubWindow): HubItem[] {
}
if (!sessions.length) {
items.push({
label: multiHost ? "(no live sessions)" : "(no live sessions; run g2mirror <command>)",
label: isTmux(control)
? "(no tmux sessions; select the host to start one)"
: multiHost
? "(no live sessions)"
: "(no live sessions; run g2mirror <command>)",
onSelect: () => control.client?.listSessions(),
});
}
@@ -1499,6 +1874,10 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
label: "Add connection",
onSelect: () => beginAddConnection(window),
},
{
label: "This device's SSH key",
onSelect: () => openClientKeyPage(window),
},
{
label: "Done",
onSelect: () => setHubMode(window, "sessions"),
@@ -1509,6 +1888,13 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
function controlStatusWord(control: ControlConnection): string {
if (!control.config.enabled) return "off";
const issue = control.sshIssue;
if (issue) {
if (issue.kind === "hostkey-unknown") return "verify host key";
if (issue.kind === "hostkey-mismatch") return "HOST KEY CHANGED";
if (issue.passwordTried) return "password rejected";
return issue.passwordAllowed ? "password needed" : "key not authorized";
}
switch (control.state?.phase) {
case "connected":
case "attached":
@@ -1541,6 +1927,32 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
connectControl(control);
},
},
...(control.config.hostKeys.length
? [
{
label: "Forget host key",
onSelect: (ctx: { stack: { pop(): unknown } }) => {
ctx.stack.pop();
control.config.hostKeys = [];
updateConnection(control.config.id, { hostKeys: [] });
control.sshIssue = null;
if (control.config.enabled) startControl(control);
renderHubWindows();
},
},
]
: []),
...(isSshConnection(control) && getSshSecret(sshPasswordSecretId(control.config.id))
? [
{
label: "Forget saved password",
onSelect: (ctx: { stack: { pop(): unknown } }) => {
ctx.stack.pop();
removeSshSecret(sshPasswordSecretId(control.config.id));
},
},
]
: []),
{
label: `Remove ${name}`,
onSelect: (ctx) => {
@@ -1558,6 +1970,11 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
windowMenu(window).open(items);
}
function isSshConnection(control: ControlConnection): boolean {
const kind = clientOptionsFor(control.config)?.kind;
return kind === "tmux" || kind === "ssh-g2mirror";
}
/** Enable and (re)connect now; also the manual retry path, so reset backoff. */
function connectControl(control: ControlConnection): void {
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
@@ -1578,6 +1995,7 @@ function disconnectControl(control: ControlConnection): void {
function removeControl(control: ControlConnection): void {
stopControl(control);
removeSshSecret(sshPasswordSecretId(control.config.id));
controls.delete(control.config.id);
saveConnections(loadConnections().filter((connection) => connection.id !== control.config.id));
renderHubWindows();
@@ -1586,7 +2004,7 @@ function removeControl(control: ControlConnection): void {
/**
* Enter the Add-connection screen: clear the draft and ask the shell to open
* the phone text editor on it. The user types (or voice-inputs) the
* g2mirror:// string, then clicks to save or double-clicks to cancel.
* connection string, then clicks to save or double-clicks to cancel.
*/
function beginAddConnection(window: HubWindow): void {
terminalNewConnectionSetting.set("");
@@ -1596,15 +2014,15 @@ function beginAddConnection(window: HubWindow): void {
function saveAddConnection(window: HubWindow): void {
const url = terminalNewConnectionSetting.get();
if (!parseConnectionString(url)) {
if (!parseAnyConnectionString(url)) {
window.addError = url
? "Not a g2mirror://token@host connection string."
: "Enter a g2mirror:// connection string first.";
? "Not an ssh://user@host or g2mirror://token@host string."
: "Enter a connection string first.";
scheduleRender(window);
return;
}
const connections = loadConnections();
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true });
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true, hostKeys: [] });
saveConnections(connections);
endAddConnection(window);
setHubMode(window, "connections");
@@ -1634,6 +2052,15 @@ function handleHubInput(window: HubWindow, event: InputEvent, frameId: number):
frameTimings.finishFrame(frameId, "discarded: terminal add-connection ignored input");
return;
}
if (window.mode === "page") {
if (event.type === "click" && window.page?.onClick) {
window.page.onClick();
renderAndSubmit(window, frameId);
return;
}
frameTimings.finishFrame(frameId, "discarded: terminal page ignored input");
return;
}
const list = hubMenu(window);
list.setItems(hubItems(window));
switch (event.type) {
@@ -1800,6 +2227,7 @@ function openViewWindow(control: ControlConnection, socket: string, label: strin
focus: true,
// Default to tall sessions; the app display setting can override this.
heightMode: "max",
acceptsKeys: true,
});
}
@@ -1820,6 +2248,11 @@ function launchPresetNames(): string[] {
*/
function launchOnHost(window: HubWindow, control: ControlConnection): void {
const presets = launchPresetNames();
if (presets.length === 0) {
// A tmux host with no presets configured: a new session running the login shell.
launchFromUi(control, "shell");
return;
}
if (presets.length === 1) {
launchFromUi(control, presets[0]!);
return;
@@ -1864,6 +2297,9 @@ function paintHub(window: HubWindow): GrayImage {
if (window.mode === "add") {
return paintAddConnection(window);
}
if (window.mode === "page" && window.page) {
return paintHubPage(window, window.page);
}
const { viewportWidth: width, viewportHeight: height, padding } = window;
const image = new GrayImage(width, height, 0);
const font = chromeFont();
@@ -1878,7 +2314,7 @@ function paintHub(window: HubWindow): GrayImage {
let listTop = header.listTop;
if (window.mode === "sessions" && controls.size === 0) {
image.drawText(font, header.titleX, listTop, "Add a g2mirror:// connection to get started, see:", 150);
image.drawText(font, header.titleX, listTop, "Add an ssh:// (tmux) or g2mirror:// connection to get started, see:", 150);
image.drawText(font, header.titleX, listTop + step, "https://github.com/jimrandomh/g2mirror", 190);
listTop += 2 * step + 6;
}
@@ -1920,8 +2356,8 @@ function paintAddConnection(window: HubWindow): GrayImage {
image.drawText(font, padding.left, padding.top, "Add connection", 220);
const bodyX = padding.left + PAGE_BODY_INDENT;
const instructionsY = padding.top + 10 + step;
image.drawText(font, bodyX, instructionsY, "Type the g2mirror://token@host string in the", 170);
image.drawText(font, bodyX, instructionsY + step, "phone app (or use voice input from the menu).", 170);
image.drawText(font, bodyX, instructionsY, "Type ssh://user@host (tmux) or g2mirror://token@host in", 170);
image.drawText(font, bodyX, instructionsY + step, "the phone app (or use voice input from the menu).", 170);
const draftY = instructionsY + 3 * step;
const draft = terminalNewConnectionSetting.get();
image.drawText(font, bodyX, draftY, truncateLabel(draft || "(empty)", 52), 220);
@@ -1932,6 +2368,34 @@ function paintAddConnection(window: HubWindow): GrayImage {
return image;
}
/** A page of text: title at the top padding, wrapped body, hint on the bottom padding. */
function paintHubPage(window: HubWindow, page: HubPage): GrayImage {
const { viewportWidth: width, viewportHeight: height, padding } = window;
const image = new GrayImage(width, height, 0);
const font = chromeFont();
const step = lineStep(font);
image.drawText(font, padding.left, padding.top, page.title, 220);
const bodyX = padding.left + PAGE_BODY_INDENT;
const hintY = height - padding.bottom - font.lineHeight;
const lines = wrapText(font, page.body(), Math.max(1, width - padding.right - bodyX), { breakLongWords: true });
let y = padding.top + 10 + step;
for (const line of lines) {
if (y + step > hintY) break;
image.drawText(font, bodyX, y, line, 190);
y += step;
}
image.drawText(font, bodyX, hintY, page.hint, 110);
return image;
}
/** The action row under a host whose SSH connection needs the user. */
function sshIssueAction(issue: SshFailure): string {
if (issue.kind === "hostkey-unknown") return "Verify host key...";
if (issue.kind === "hostkey-mismatch") return "Host key changed!";
if (issue.passwordAllowed) return issue.passwordTried ? "Re-enter password..." : "Enter password...";
return "Show this device's SSH key...";
}
function truncateLabel(text: string, maxLength: number): string {
return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`;
}
+44
View File
@@ -0,0 +1,44 @@
import type {
G2MirrorHistoryReply,
G2MirrorPhase,
G2MirrorSession,
G2MirrorState,
} from "../../native/terminal-session-types";
/**
* What the Terminal worker needs from a session backend. G2MirrorClient (a
* g2mirror server, directly or tunnelled over SSH) and TmuxClient (tmux
* control mode over SSH) both provide it. The model is g2mirror's: a client
* starts "connecting", reaches "connected" (it can list and launch sessions,
* and reports bells/activity/titles for them), and can then attach to one
* session, view it (snapshot + live output), page its scrollback, and type
* into it.
*/
export interface TerminalClient {
state(): G2MirrorState;
onStateChange(listener: (state: G2MirrorState) => void): () => void;
/** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
onTerminalData(listener: (data: Uint8Array, kind: "snapshot" | "output") => void): () => void;
onSessionAttached(listener: (command: string) => void): () => void;
onSessionDetached(listener: (reason: string) => void): () => void;
onBell(listener: (socket: string, lastBellAtMs: number) => void): () => void;
onActivity(listener: (socket: string, lastOutputAtMs: number) => void): () => void;
onTitle(listener: (socket: string, title: string) => void): () => void;
/** Fired after a snapshot's terminal data, with its scrollback splice point. */
onSnapshot(listener: (historyNext: number, historyOldest: number) => void): () => void;
onHistoryLines(listener: (reply: G2MirrorHistoryReply) => void): () => void;
setViewport(cols: number, rows: number): void;
start(): void;
stop(): void;
listSessions(): void;
launchSession(preset: string): Promise<string>;
connectSession(socket: string): void;
disconnectSession(): void;
view(): void;
unview(): void;
requestHistory(before: number, limit?: number): void;
sendInput(text: string): void;
submitInput(text: string): void;
}
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
+5
View File
@@ -82,4 +82,9 @@ export class TerminalEmulator {
cursorCol(): number {
return this.term.buffer.active.cursorX;
}
/** DECCKM: the session asked for application cursor keys (ESC O A rather than ESC [ A). */
applicationCursorKeys(): boolean {
return this.term.modes.applicationCursorKeysMode;
}
}
+118
View File
@@ -0,0 +1,118 @@
/**
* What a terminal emulator sends for a key from a hardware keyboard, by the
* classic xterm conventions: Ctrl chords become control characters, Alt
* prefixes ESC, and modified cursor and function keys carry the xterm
* modifier parameter (1 + shift + 2*alt + 4*ctrl). Follows the Glassly
* Terminal miniapp's encoder (terminal/keys.ts, MIT).
* No platform imports: this runs under node in the tests.
*/
import type { KeyEvent } from "../../g2/keyboard/keys";
const ESC = "\x1b";
/** Keys xterm encodes as `ESC [ <n> ~`, with a modifier as `;m`. */
const TILDE_KEYS: Record<string, number> = {
Insert: 2,
Delete: 3,
PageUp: 5,
PageDown: 6,
F5: 15,
F6: 17,
F7: 18,
F8: 19,
F9: 20,
F10: 21,
F11: 23,
F12: 24,
};
/** Keys with a single-letter final byte: SS3 in application cursor mode, CSI otherwise. */
const LETTER_KEYS: Record<string, string> = {
ArrowUp: "A",
ArrowDown: "B",
ArrowRight: "C",
ArrowLeft: "D",
Home: "H",
End: "F",
};
/** F1-F4 are always SS3 (`ESC O P`), or CSI with a modifier. */
const SS3_KEYS: Record<string, string> = { F1: "P", F2: "Q", F3: "R", F4: "S" };
/** Ctrl with a punctuation or digit key, per xterm (^[ ^\ ^] ^^ ^_ ^? ^@). */
const CONTROL_PUNCTUATION: Record<string, string> = {
"[": "\x1b",
"\\": "\x1c",
"]": "\x1d",
"^": "\x1e",
"6": "\x1e",
"_": "\x1f",
"-": "\x1f",
"?": "\x7f",
"/": "\x7f",
"8": "\x7f",
"@": "\x00",
"2": "\x00",
};
/**
* The text to write to the terminal for one key event, or null when the key
* sends nothing: releases, bare modifiers, Meta chords, and keys a terminal
* has no bytes for (volume, media). `appCursor` is DECCKM, which the emulator
* tracks from the session's output.
*/
export function encodeTerminalKey(event: KeyEvent, appCursor: boolean): string | null {
if (event.phase !== "down") return null;
const { key, modifiers } = event;
if (modifiers.meta) return null;
const modifierParam = 1 + (modifiers.shift ? 1 : 0) + (modifiers.alt ? 2 : 0) + (modifiers.ctrl ? 4 : 0);
const modified = (prefix: string, final: string): string | null =>
modifierParam > 1 ? `${ESC}[${prefix};${modifierParam}${final}` : null;
switch (key) {
case "Enter":
return modifiers.alt ? `${ESC}\r` : "\r";
case "Escape":
return ESC;
case "Tab":
return modifiers.shift ? `${ESC}[Z` : "\t";
case "Backspace":
if (modifiers.alt) return `${ESC}\x7f`;
return modifiers.ctrl ? "\x08" : "\x7f";
case " ":
return modifiers.ctrl ? "\x00" : modifiers.alt ? `${ESC} ` : " ";
}
const letter = LETTER_KEYS[key];
if (letter) return modified("1", letter) ?? (appCursor ? `${ESC}O${letter}` : `${ESC}[${letter}`);
const tilde = TILDE_KEYS[key];
if (tilde) return modified(String(tilde), "~") ?? `${ESC}[${tilde}~`;
const ss3 = SS3_KEYS[key];
if (ss3) return modified("1", ss3) ?? `${ESC}O${ss3}`;
if (modifiers.ctrl) {
// From the unshifted key: ^A..^Z, then the punctuation set.
const base = key.length === 1 ? key.toLowerCase() : "";
let char: string | undefined;
if (base >= "a" && base <= "z") char = String.fromCharCode(base.charCodeAt(0) - 96);
else char = CONTROL_PUNCTUATION[key];
if (char === undefined) return null;
return modifiers.alt ? `${ESC}${char}` : char;
}
// With Alt held the translator leaves `text` out (it is a chord, not
// typing); a terminal still wants ESC and the character.
const text = event.text ?? (modifiers.alt && key.length === 1 ? key : undefined);
if (text === undefined) return null;
return modifiers.alt ? `${ESC}${text}` : text;
}
/** Shift+PageUp / Shift+PageDown page the emulator's own scrollback, as in xterm. */
export function terminalScrollbackKey(event: KeyEvent): -1 | 1 | null {
if (event.phase !== "down") return null;
const { shift, ctrl, alt, meta } = event.modifiers;
if (!shift || ctrl || alt || meta) return null;
if (event.key === "PageUp") return -1;
if (event.key === "PageDown") return 1;
return null;
}
File diff suppressed because it is too large. Load diff
+29 -2
View File
@@ -1,5 +1,32 @@
import { type AppDefinition } from "../app-definition";
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
import { type AppContext, type AppDefinition } from "../app-definition";
import {
createTranscribeAppWindow,
createTranscribeReplayWindow,
TRANSCRIBE_SURFACE_ID,
TRANSCRIBE_WINDOW_ID,
} from "./transcribe-app";
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
let nextReplaySerial = 1;
/**
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
* own. Returns false where recordings can't be replayed.
*/
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
if (!ctx.actions.openVoiceReplay) return false;
const windowId = `transcribe:replay:${nextReplaySerial++}`;
void ctx
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
const window = createTranscribeReplayWindow(windowId, path, name, options);
if (!window) throw new Error("recordings can't be replayed here");
return window;
})
.catch((error) => {
ctx.appendLog(`transcribe replay window failed: ${error}`);
});
return true;
}
const transcribeApp: AppDefinition = {
appId: "transcribe",
+23
View File
@@ -0,0 +1,23 @@
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
export function formatReplayTime(ms: number, withHours = false): string {
const total = Math.max(0, Math.floor(ms / 1000));
const hours = Math.floor(total / 3600);
const minutes = Math.floor((total % 3600) / 60);
const seconds = String(total % 60).padStart(2, "0");
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
}
/** The time picker's fields, in display and click order. */
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
export type TimeField = (typeof TIME_FIELDS)[number];
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
/**
* Step a position (whole seconds) by `delta` of a field's unit, carrying
* into the neighboring fields, within [0, maxSeconds].
*/
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
}
+147
View File
@@ -0,0 +1,147 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
import { CONTEXT_MENU_DIM } from "../../ui/menu";
import { centeredTextY } from "../../ui/metrics";
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
const PADDING = 12;
const FIELD_GAP = 6;
const FIELD_PAD_X = 6;
/** Fill and outline of the active field (the Timers and date pickers' field style). */
const ACTIVE_FILL = 35;
const ACTIVE_STROKE = 130;
export type TimePickerOptions = {
title: string;
/** Where the picker starts, and the furthest it goes, in ms. */
initialMs: number;
maxMs: number;
onPick: (positionMs: number) => void;
};
/**
* A modal position picker for a recording: hours (when the recording has
* any), minutes and seconds, over the dimmed content beneath. Input follows
* the date picker's: scroll adjusts the active field (up is later, carrying
* into the fields beside it), click moves to the next field and on the last
* one picks, double-click steps back a field and on the first one cancels.
* A watch's swipes move between fields and adjust.
*/
export class TimePickerLayer implements Layer {
readonly acceptsDirectional = true;
readonly dimUnderneath = CONTEXT_MENU_DIM;
private readonly fields: readonly TimeField[];
private readonly maxSeconds: number;
private seconds: number;
private field = 0;
constructor(private readonly options: TimePickerOptions) {
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
}
handleInput(event: InputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
case "swipe-up":
this.adjust(1);
return;
case "scroll-down":
case "swipe-down":
this.adjust(-1);
return;
case "swipe-right":
this.field = Math.min(this.fields.length - 1, this.field + 1);
return;
case "swipe-left":
this.field = Math.max(0, this.field - 1);
return;
case "click":
if (this.field < this.fields.length - 1) {
this.field++;
} else {
ctx.stack.pop();
this.options.onPick(this.seconds * 1000);
}
return;
case "double-click":
if (this.field > 0) this.field--;
else ctx.stack.pop();
return;
default:
return;
}
}
private adjust(delta: number): void {
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
}
private hints(field: number): string {
return gestureHints([
[GESTURE_SCROLL, "adjust"],
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
]);
}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
const image = paintBelow();
const small = getDefaultSmallFont();
const medium = getDefaultMediumFont();
const focused = ctx.stack.isFocused();
const base = ctx.stack.getBaseSize();
const values = [
String(Math.floor(this.seconds / 3600)),
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
String(this.seconds % 60).padStart(2, "0"),
].slice(TIME_FIELDS.length - this.fields.length);
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
const fieldWidths = this.fields.map(
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
);
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
const boxHeight = medium.lineHeight + 6;
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
const width = Math.min(
base.width,
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
);
const titleHeight = small.lineHeight + 12;
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
const x = Math.max(0, Math.floor((base.width - width) / 2));
const y = Math.max(0, Math.floor((base.height - height) / 2));
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
image.fillRoundedRect(x, y, width, height, 1);
image.drawRoundedRect(x, y, width, height, 72);
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
const top = y + 8 + titleHeight;
let fieldX = x + PADDING;
for (let index = 0; index < this.fields.length; index++) {
const fieldWidth = fieldWidths[index]!;
const active = index === this.field;
if (active) {
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
}
const value = values[index]!;
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
const caption = this.fields[index]!;
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
top + boxHeight + 2, caption, active ? 190 : 110);
fieldX += fieldWidth + FIELD_GAP;
}
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
return image;
}
}
+101 -4
View File
@@ -1,8 +1,11 @@
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
import { voiceControlBridge } from "../../native/voice-control";
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { type ContinuousCaptureOptions } from "../../ui/layers";
import { withRecordingIcon } from "../../ui/listening-header";
import { TimePickerLayer } from "./time-picker";
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
}
export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
stopContinuousVoiceCapture: () => void;
};
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
// One timestamp per session names both the audio recording and a saved transcript.
const stamp = transcriptTimestamp();
const layer = new TranscribeLayer(stamp);
// At launch, and again when the Microphones setting changes: a running
// capture moves over to the microphones now asked for.
const startCapture = () =>
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
const app = createInProcessWindow({
appId: "transcribe",
windowId: TRANSCRIBE_WINDOW_ID,
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
},
// The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
app.requestRender();
});
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
startCapture();
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
return app;
}
/**
* Transcribe in replay mode: a window of its own that transcribes a
* recording (a .mka file from Save Recording) in place of the microphones,
* in real time and through the same pipeline. It neither uses the mic nor
* shows the mic tray icon, so it runs beside a live Transcribe window. The
* header shows the position in the recording; a tap pauses and resumes; the
* menu jumps to a time.
*/
export function createTranscribeReplayWindow(
windowId: string,
path: string,
name: string,
options: InProcessAppOptions,
): InProcessWindow | null {
const replay = options.actions.openVoiceReplay?.(path) ?? null;
if (!replay) return null;
// Names a saved transcript after the recording it came from.
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
const app = createInProcessWindow({
appId: "transcribe",
windowId,
title: name,
iconLetter: "Tr",
icon: "mic",
closeable: true,
// Relaunching Transcribe after a restart would open the microphone, not this recording.
restoresApp: false,
// Like a playing video: something to watch, not to touch.
keepsScreenOn: () => replay.state === "playing",
actions: options.actions,
baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [
{
label: "Jump to time",
// The length isn't known until the recording has been opened.
disabled: () => replay.durationMs <= 0,
onSelect: (ctx) => {
ctx.stack.pop();
ctx.stack.push(
new TimePickerLayer({
title: "Jump to time",
initialMs: replay.positionMs,
maxMs: replay.durationMs,
onPick: (positionMs) => {
replay.jumpTo(positionMs);
app.requestRender();
},
}),
);
},
},
{
label: replay.state === "paused" ? "Resume" : "Pause",
disabled: () => replay.state !== "playing" && replay.state !== "paused",
onSelect: (ctx) => {
ctx.stack.pop();
replay.setPaused(replay.state === "playing");
},
},
{
label: "Restart from 0:00",
onSelect: (ctx) => {
ctx.stack.pop();
replay.jumpTo(0);
},
},
{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
},
},
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
replay.close();
options.onClosed();
},
});
// Started first, so the layer picks its opening status up as it subscribes.
replay.start();
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
return app;
}
@@ -0,0 +1,40 @@
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
import { type MicCount } from "../../ui/layers";
import { micControlSupported } from "../microphones/mic-control";
export type TranscribeMics = "1" | "2" | "4";
const MICS_LABELS: Record<TranscribeMics, string> = {
"1": "1 (mixed)",
"2": "2 (one per temple)",
"4": "4 (two per temple)",
};
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
export function separateMicsSupported(): boolean {
return glassesTypeSetting.get() === "g2" && micControlSupported();
}
/**
* How many of the glasses' microphones Transcribe captures. With more than
* one, the temples stream their own microphones (each temple on its own
* clock) instead of the glasses' mixed stream; a recording keeps every one,
* and what is transcribed is a mix made on the phone.
*/
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
id: "transcribe-mics",
label: "Microphones",
storageKey: "transcribe.mics",
defaultValue: "1",
values: ["1", "2", "4"],
formatValue: (value) => MICS_LABELS[value],
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
description:
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
});
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
export function transcribeMicCount(): MicCount {
if (!separateMicsSupported()) return 1;
return Number(transcribeMicsSetting.get()) as MicCount;
}
+72 -13
View File
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
import { type InputEvent } from "../../ui/gestures";
import { formatReplayTime } from "./replay-time";
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
/**
* Live transcription view. Continuous mic capture is owned by the app wrapper
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
* it, and saves it to Downloads from the app menu.
*
* Given a replay, it transcribes that recording instead of the microphone:
* the header shows the position in the recording, a tap pauses and resumes,
* and each jump leaves its time as a line in the transcript.
*/
export class TranscribeLayer implements Layer {
private status = "Starting microphone...";
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribePause: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeReplay: (() => void) | null = null;
private readonly source: VoiceEventSource;
/** `stamp` names the saved transcript, matching the session's audio recording. */
constructor(private readonly stamp: string = transcriptTimestamp()) {}
/**
* `stamp` names the saved transcript, matching the session's audio
* recording. `replay` is the recording to transcribe in place of the
* microphone; `replayName` is what to call it.
*/
constructor(
private readonly stamp: string = transcriptTimestamp(),
private readonly replay: VoiceReplay | null = null,
private readonly replayName = "",
) {
this.source = replay ? replay.events : voiceControlBridge;
if (replay) this.status = "Opening the recording...";
}
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
start(requestRender: () => void, isVisible: () => boolean): void {
this.meter.start(requestRender, isVisible);
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.meter.start(requestRender, isVisible, this.source);
this.unsubscribeTranscript = this.source.onTranscript((event) => {
this.transcript.accept(event);
requestRender();
});
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
// The current status arrives while subscribing, before the window has a
// surface to render to; the window paints it when it is first shown.
let subscribing = true;
this.unsubscribeStatus = this.source.onStatus((state) => {
this.status = state.status;
this.listening = state.listening;
this.detail = state.detail;
this.saveNotice = "";
requestRender();
if (!subscribing) requestRender();
});
subscribing = false;
const replay = this.replay;
if (replay) {
let shownSecond = -1;
let shownState = replay.state;
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
// The clock ticks in whole seconds; repaint when what it shows changed.
const second = Math.floor(replay.positionMs / 1000);
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
shownSecond = second;
shownState = replay.state;
if (isVisible()) requestRender();
});
}
}
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
private replayStatus(replay: VoiceReplay): string {
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
const withHours = replay.durationMs >= 3_600_000;
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
if (replay.state === "paused") return `${clock} · Paused`;
if (replay.state === "ended") return `${clock} · Finished`;
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
return this.listening ? clock : `${clock} · ${this.status}`;
}
paint(ctx: LayerContext): GrayImage {
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
const viewport = ctx.stack.getBaseSize();
const image = new GrayImage(viewport.width, viewport.height, 0);
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
const replay = this.replay;
const status = replay
? this.replayStatus(replay)
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
const playing = !replay || replay.state === "playing";
const bodyTop = paintListeningHeader(
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
voiceControlBridge.recording,
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
this.source.recording,
);
const step = lineStep(font) + 2;
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
ctx.actions.requestRender();
return;
}
if (event.type === "click" && this.replay) {
this.replay.setPaused(this.replay.state === "playing");
ctx.actions.requestRender();
return;
}
if (event.type === "double-click") {
ctx.stack.pop();
}
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
onRemoved(): void {
this.meter.stop();
this.unsubscribeReplay?.();
this.unsubscribeReplay = null;
this.unsubscribePause?.();
this.unsubscribePause = null;
this.unsubscribeTranscript?.();
+12
View File
@@ -36,6 +36,18 @@ export class TranscriptModel {
if (this.live) this.breakAfterLive = true;
else if (this.finalized) this.separator = "\n";
}
/**
* A line of its own (where a replay jumped to), ending whatever was in
* progress: what follows it starts a new paragraph.
*/
mark(label: string): void {
const before = this.text;
this.finalized = before ? `${before}\n${label}` : label;
this.live = "";
this.separator = "\n";
this.breakAfterLive = false;
}
}
/** An absolute history position, with an explicit follow-tail mode. */