Add a video player, launched by selecting a video file from the Files app

This commit is contained in:
James Babcock committed 2026-09-28 14:11:04 -07:00
1 parent deaa93fead
commit cb6dddd120
13 files changed
+1784 -3

No files matched your search

+18 -2
View File
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
import { isFontFile } from "../../native/font-files";
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
import { isDecodableImageFile } from "../../native/image-files";
import { isPlayableVideoFile } from "../../native/video-player";
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
import { FileBrowserLayer } from "./file-browser";
@@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & {
installEhpkApp: (path: string) => Promise<void> | void;
/** Open a font file's previewer as its own shell window. */
openFontWindow: (title: string, path: string) => void;
/** Open a video file's player (settings panel first) as its own shell window. */
openVideoWindow: (title: string, path: string) => void;
};
/**
* The Files app's launcher-opened window: a file browser over Places
* (bookmarks and storage roots). Picking any file opens an info dialog with
* metadata plus, for viewable types (text, images), the open actions.
* metadata plus, for viewable types (text, images, video), the open actions.
*/
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
let created: InProcessWindow | null = null;
const browser = new FileBrowserLayer({
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)),
isSupportedFile: (name) =>
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
// The browser handles double-click itself (up a level), so it is not
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
onLeave: () => shell.yieldFocusToSidebar(),
@@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
},
];
}
if (isPlayableVideoFile(entry.name)) {
return [
{
label: "Play video",
onSelect: (ctx) => {
ctx.stack.pop();
options.openVideoWindow(entry.name, entry.path);
},
},
];
}
if (isFontFile(entry.name)) {
// Install copies into the app-internal fonts directory, making the face
// available in the Settings font pickers. The row relabels itself with
+13
View File
@@ -13,6 +13,7 @@ import {
FILES_SURFACE_ID,
FILES_WINDOW_ID,
} from "./files-app";
import { createVideoPlayerWindow } from "./video-window";
// Document windows are closeable and non-singleton; the serial keeps their
// windowIds unique.
@@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v
});
}
function openVideoWindow(ctx: AppContext, title: string, path: string): void {
const windowId = `files:doc:${nextDocumentSerial++}`;
void ctx
.launchInProcessApp(windowId, `window:${windowId}`, (options) =>
createVideoPlayerWindow(windowId, title, path, options),
)
.catch((error) => {
ctx.appendLog(`video window failed: ${error}`);
});
}
const filesApp: AppDefinition = {
appId: "files",
title: "Files",
@@ -78,6 +90,7 @@ const filesApp: AppDefinition = {
}
},
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
}),
),
// A document arriving via Android's Share intent opens as its own window.
+86
View File
@@ -0,0 +1,86 @@
/**
* Where a video goes in the player window, for each size choice. Pure
* geometry, so it is unit-tested without the NativeScript runtime.
*/
export type VideoSize = "small" | "medium" | "large" | "full";
export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"];
export type Size = { width: number; height: number };
export type Rect = Size & { x: number; y: number };
/**
* Bounding boxes for the fixed sizes; a video is fitted inside, keeping its
* aspect. Large and full fit the whole viewport instead (full in a tall
* window). Bigger pictures cost the glasses link more per frame, so smaller
* sizes play at a higher frame rate.
*/
const FIXED_BOXES: Partial<Record<VideoSize, Size>> = {
small: { width: 160, height: 120 },
medium: { width: 240, height: 180 },
};
export function videoSizeLabel(size: VideoSize): string {
switch (size) {
case "small":
return "Small";
case "medium":
return "Medium";
case "large":
return "Large";
case "full":
return "Full screen";
}
}
/** Full screen needs the tall window band; the rest play in the standard one. */
export function videoHeightMode(size: VideoSize): "min" | "max" {
return size === "full" ? "max" : "min";
}
/**
* The largest size with the video's aspect ratio that fits in box, with even
* dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel
* columns) of at least 2. Upscales small videos too: the size is the
* viewer's choice.
*/
export function fitVideo(video: Size, box: Size): Size {
if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) };
const scale = Math.min(box.width / video.width, box.height / video.height);
return {
width: evenFloor(Math.min(box.width, Math.round(video.width * scale))),
height: evenFloor(Math.min(box.height, Math.round(video.height * scale))),
};
}
/**
* The video's rect within a viewport, centered. x snaps to a multiple of 4
* and y to a multiple of 2 so frame deltas don't straddle an extra encoder
* block at each edge.
*/
export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect {
const fixed = FIXED_BOXES[size];
const box = fixed
? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) }
: viewport;
const fitted = fitVideo(video, box);
return {
x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4),
y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2),
...fitted,
};
}
/** "3:52", or "1:02:03" past an hour. */
export function formatMediaTime(ms: number): string {
const total = Math.max(0, Math.floor(ms / 1000));
const hours = Math.floor(total / 3600);
const minutes = Math.floor((total % 3600) / 60);
const seconds = String(total % 60).padStart(2, "0");
return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`;
}
function evenFloor(value: number): number {
return Math.max(2, Math.floor(value / 2) * 2);
}
+443
View File
@@ -0,0 +1,443 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { PHOTO_GAMMA } from "../../native/image-files";
import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player";
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
import { type InputEvent } from "../../ui/gestures";
import { type Layer, type LayerContext } from "../../ui/layers";
import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu";
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { appViewportSize } from "../../ui/shell/geometry";
import {
formatMediaTime,
VIDEO_SIZES,
videoHeightMode,
videoRect,
videoSizeLabel,
type Rect,
type VideoSize,
} from "./video-layout";
export const videoSizeSetting = new ConfigSettingEnum<VideoSize>({
id: "video-size",
label: "Size",
storageKey: "video.size",
defaultValue: "large",
values: VIDEO_SIZES,
formatValue: videoSizeLabel,
});
export const videoAudioSetting = new ConfigSettingBoolean({
id: "video-audio",
label: "Audio",
storageKey: "video.audio",
defaultValue: true,
description: "Play the video's sound on the phone's current audio output.",
});
export const videoMonochromeSetting = new ConfigSettingBoolean({
id: "video-monochrome",
label: "Monochrome",
storageKey: "video.monochrome",
defaultValue: false,
description: "Show only black and white: far less to send per frame, so a higher frame rate.",
});
/** How far one ring scroll seeks. */
const SEEK_STEP_MS = 10_000;
const MARGIN_X = 18;
const TITLE_Y = 10;
const PANEL_WIDTH = 340;
/**
* One opened video: its probe result, the native player, and the window
* glue. The window's base layer is the settings panel; playing pushes the
* playback layer, and backing out of it (or the video ending) returns to the
* panel with the position kept for Resume.
*/
class VideoSession {
readonly probe: VideoInfo | { error: string };
private player: VideoPlayer | null = null;
private window: InProcessWindow | null = null;
private playback: VideoPlaybackLayer | null = null;
/** Where Resume starts; 0 = from the beginning. */
resumeMs = 0;
/** The last playback error, shown on the panel until the next Play. */
error: string | null = null;
constructor(
readonly path: string,
readonly title: string,
readonly surfaceId: string,
) {
this.probe = probeVideo(path);
}
get info(): VideoInfo | null {
return "error" in this.probe ? null : this.probe;
}
attach(window: InProcessWindow): void {
this.window = window;
}
isPlaying(): boolean {
return this.player?.isPlaying() ?? false;
}
positionMs(): number {
return this.player?.positionMs() ?? this.resumeMs;
}
/** Current rect for a size choice, in the viewport of the band that size plays in. */
rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect {
const info = this.info;
return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport);
}
play(ctx: LayerContext, fromStart: boolean): void {
const window = this.window;
const info = this.info;
if (!window || !info) return;
this.error = null;
const size = videoSizeSetting.get();
window.setHeightMode(videoHeightMode(size));
const viewport = ctx.stack.getBaseSize();
const rect = this.rectFor(size, viewport);
const player = this.ensurePlayer();
this.playback = new VideoPlaybackLayer(this, rect, viewport);
ctx.stack.push(this.playback);
player.start({
path: this.path,
surfaceId: this.surfaceId,
rect,
monochrome: videoMonochromeSetting.get(),
gamma: PHOTO_GAMMA,
audio: info.hasAudio && videoAudioSetting.get(),
startMs: fromStart ? 0 : this.resumeMs,
});
}
togglePause(): void {
const player = this.player;
if (!player) return;
if (player.isPlaying()) player.pause();
else player.resume();
}
seekBy(deltaMs: number): void {
const player = this.player;
if (!player) return;
const duration = this.info?.durationMs ?? 0;
const target = Math.max(0, player.positionMs() + deltaMs);
player.seekTo(duration > 0 ? Math.min(target, duration) : target);
}
/** The window went to the background or the screen went off: stop, keep the place. */
pauseForHost(): void {
if (!this.player?.isPlaying()) return;
this.player.pause();
// Paint the paused overlay now, so the retained surface shows it on return.
this.window?.requestRender();
}
/** Leave the playback layer for the panel, remembering where to resume. */
returnToPanel(resumeMs: number): void {
const player = this.player;
if (player) {
player.pause();
player.stop();
}
this.resumeMs = resumeMs;
const window = this.window;
const playback = this.playback;
this.playback = null;
if (window && playback) window.stack.popThrough(playback);
window?.setHeightMode("min");
window?.requestRender();
}
lastFrame(rect: Rect): GrayImage | null {
return this.player?.lastFrame(rect.width, rect.height) ?? null;
}
release(): void {
this.player?.release();
this.player = null;
}
private ensurePlayer(): VideoPlayer {
if (!this.player) {
this.player = new VideoPlayer({
onState: (state, positionMs) => this.onPlayerState(state, positionMs),
onError: (message) => {
this.error = message;
this.returnToPanel(this.positionMs());
},
});
}
return this.player;
}
private onPlayerState(state: VideoPlaybackState, _positionMs: number): void {
if (state === "ended") {
this.returnToPanel(0);
return;
}
// A paused seek decoded its new frame: repaint it under the overlay.
if (state === "paused") this.window?.requestRender();
}
}
type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean };
/**
* The pre-play settings panel (the window's root): picture size, audio and
* monochrome, all remembered for the next video, then Play (or Resume plus
* Play from start once a position is kept).
*/
class VideoSettingsLayer implements Layer {
private readonly menu = new Menu<PanelRow>({
wrap: false,
rowGap: 1,
highlight: { radius: 6 },
getHeight: () => listRowHeight(getDefaultSmallFont()),
draw: (args) => this.drawRow(args),
});
constructor(private readonly session: VideoSession) {}
private rows(): PanelRow[] {
if (!this.session.info) return [];
const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }];
rows.push({ kind: "play", fromStart: false });
if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true });
return rows;
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220);
const step = lineStep(font);
let y = TITLE_Y + step;
const info = this.session.info;
const errors: string[] = [];
if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`);
if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`);
if (info) {
const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"];
image.drawText(font, MARGIN_X, y, facts.join(" · "), 130);
y += step;
}
for (const error of errors) {
for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) {
image.drawText(font, MARGIN_X, y, line, 200);
y += step;
}
}
y += 6;
const rows = this.rows();
const selected = this.menu.selectedItem;
const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1;
// Land on Resume/Play by default: the settings usually stay as they were.
const playIndex = rows.findIndex((row) => row.kind === "play");
this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined);
const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6));
this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) },
ctx.stack.isFocused());
return image;
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
switch (event.type) {
case "scroll-up":
case "scroll-down":
await this.menu.handleInput(event);
return;
case "click": {
const row = this.menu.selectedItem;
if (row) this.activate(row, ctx);
return;
}
default:
return;
}
}
private activate(row: PanelRow, ctx: LayerContext): void {
switch (row.kind) {
case "size":
videoSizeSetting.set(videoSizeSetting.next());
return;
case "audio":
if (this.session.info?.hasAudio) videoAudioSetting.toggle();
return;
case "monochrome":
videoMonochromeSetting.toggle();
return;
case "play":
this.session.play(ctx, row.fromStart);
return;
}
}
private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs<PanelRow>): void {
const font = getDefaultSmallFont();
const textX = x + 10;
const textWidth = width - 20;
switch (item.kind) {
case "size": {
const size = videoSizeSetting.get();
// The picture size this choice gets in its own band (full screen plays in the tall window).
const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size)));
drawRightValueMenuItem(image, font, textX, y, textWidth, "Size",
`${videoSizeLabel(size)} ${rect.width}x${rect.height}`);
return;
}
case "audio": {
const hasAudio = this.session.info?.hasAudio ?? false;
drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)",
hasAudio && videoAudioSetting.get(), selected);
return;
}
case "monochrome":
drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected);
return;
case "play": {
const resumeMs = this.session.resumeMs;
const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play";
image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200);
return;
}
}
}
}
function sameRow(a: PanelRow, b: PanelRow): boolean {
return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart);
}
/**
* The playing video. The native player draws frames into the video rect on
* its own; this layer paints only when the window renders (input, pause,
* focus): black around the latest frame, plus a status box while paused.
* Click pauses/resumes, scroll seeks 10s, double-click returns to the panel.
*/
class VideoPlaybackLayer implements Layer {
private leaving = false;
constructor(
private readonly session: VideoSession,
private readonly rect: Rect,
/** The viewport the rect was laid out in. */
private readonly viewport: { width: number; height: number },
) {}
paint(ctx: LayerContext): GrayImage {
const { width, height } = ctx.stack.getBaseSize();
if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) {
// The window was relaid out (display mode change) and the native player's rect no
// longer fits the surface: back out to the panel, which lays out afresh on Play.
this.leaving = true;
setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0);
}
const image = new GrayImage(width, height, 0);
const frame = this.session.lastFrame(this.rect);
if (frame) image.bitBlt(frame, this.rect.x, this.rect.y);
if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null);
return image;
}
async handleInput(event: InputEvent): Promise<void> {
switch (event.type) {
case "click":
this.session.togglePause();
return;
case "scroll-up":
this.session.seekBy(-SEEK_STEP_MS);
return;
case "scroll-down":
this.session.seekBy(SEEK_STEP_MS);
return;
case "double-click":
this.session.returnToPanel(this.session.positionMs());
return;
default:
return;
}
}
/**
* "Paused 1:23 / 3:52" over a progress bar, below the picture when there is
* room (small sizes), else over its bottom edge.
*/
private drawPausedBox(image: GrayImage, loading: boolean): void {
const font = getDefaultSmallFont();
const duration = this.session.info?.durationMs ?? 0;
const position = this.session.positionMs();
const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`;
const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24);
const boxHeight = font.lineHeight + 14;
const rect = this.rect;
const x = Math.round(rect.x + (rect.width - boxWidth) / 2);
const below = rect.y + rect.height + 4;
const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6);
image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6);
image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6);
image.drawText(font, x + 12, y + 5, text, 220);
if (!loading && duration > 0) {
const barWidth = boxWidth - 24;
image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40);
image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200);
}
}
}
/** A video file's player as its own window, opened from the Files app. */
export function createVideoPlayerWindow(
windowId: string,
title: string,
path: string,
options: InProcessAppOptions,
): InProcessWindow {
const session = new VideoSession(path, title, `window:${windowId}`);
const window = createInProcessWindow({
appId: "files",
windowId,
title,
iconLetter: "V",
icon: "film",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)),
// A playing video is the thing being watched: no idle screen-off.
keepsScreenOn: () => session.isPlaying(),
onForegroundChanged: (foreground) => {
if (!foreground) session.pauseForHost();
},
setScreenOn: (on) => {
if (!on) session.pauseForHost();
},
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
reconfigureSurface: options.reconfigureSurface,
onClosed: () => {
session.release();
options.onClosed();
},
});
session.attach(window);
return window;
}
+148
View File
@@ -0,0 +1,148 @@
/**
* Bridge to the Android FaceclawVideoPlayer: decodes a video file and
* streams its frames, scaled to gray, straight into a rect of a window's
* compositor surface, with the audio on the phone's default media route.
* Android only; on iOS probeVideo reports an error and nothing plays.
*/
import { Utils } from "@nativescript/core";
import { GrayImage } from "../graphics/image";
declare const com: any;
declare const java: any;
declare const global: any;
/** Containers MediaExtractor opens (AVI is not one of them). */
const PLAYABLE_VIDEO = /\.(mp4|m4v|mkv|webm|3gp|mov)$/i;
export function isPlayableVideoFile(name: string): boolean {
return PLAYABLE_VIDEO.test(name);
}
export type VideoInfo = {
/** Display size, rotation already applied. */
width: number;
height: number;
durationMs: number;
hasAudio: boolean;
};
/** Read a video's size, duration and tracks; an error string when it can't be played. */
export function probeVideo(path: string): VideoInfo | { error: string } {
if (!global.isAndroid) return { error: "Video playback is not supported on this phone" };
try {
const raw = JSON.parse(String(com.faceclaw.app.FaceclawVideoPlayer.probe(path)));
if (raw.error) return { error: String(raw.error) };
return {
width: Number(raw.width) || 0,
height: Number(raw.height) || 0,
durationMs: Number(raw.durationMs) || 0,
hasAudio: Boolean(raw.hasAudio),
};
} catch (error) {
return { error: String(error) };
}
}
export type VideoPlaybackState = "playing" | "paused" | "ended";
export type VideoPlayerCallbacks = {
/** State change, or a paused seek that landed on a new frame (state "paused"). */
onState: (state: VideoPlaybackState, positionMs: number) => void;
onError: (message: string) => void;
};
export type VideoStartOptions = {
path: string;
surfaceId: string;
/** The video's rect in surface coordinates. */
rect: { x: number; y: number; width: number; height: number };
monochrome: boolean;
/** Tone curve for gray output (ignored for monochrome); see PHOTO_GAMMA. */
gamma: number;
audio: boolean;
startMs: number;
};
/** One player; frames go to whatever surface the latest start() named. Main thread only. */
export class VideoPlayer {
private readonly player: any;
// The Java listener proxy must stay referenced or it gets GC'd.
private readonly listenerProxy: any;
// Reused direct buffer (and its JS view) for copyLastFrame; see java-direct-buffer.ts
// for why frames cross as a Java-allocated buffer.
private frameBuffer: any = null;
private frameView: Uint8Array | null = null;
constructor(callbacks: VideoPlayerCallbacks) {
this.player = new com.faceclaw.app.FaceclawVideoPlayer(Utils.android.getApplicationContext());
this.listenerProxy = new com.faceclaw.app.FaceclawVideoPlayer.Listener({
onStateChanged: (state: string, positionMs: number) => {
callbacks.onState(String(state) as VideoPlaybackState, Number(positionMs));
},
onError: (message: string) => callbacks.onError(String(message ?? "Playback failed")),
});
this.player.setListener(this.listenerProxy);
}
start(options: VideoStartOptions): void {
const { rect } = options;
this.player.start(
options.path,
options.surfaceId,
Math.round(rect.x),
Math.round(rect.y),
Math.round(rect.width),
Math.round(rect.height),
options.monochrome,
options.gamma,
options.audio,
Math.max(0, Math.round(options.startMs)),
);
}
/** Once this returns, no more frames reach the surface until resume(). */
pause(): void {
this.player.pause();
}
resume(): void {
this.player.resume();
}
seekTo(positionMs: number): void {
this.player.seekTo(Math.max(0, Math.round(positionMs)));
}
positionMs(): number {
return Number(this.player.getPositionMs());
}
isPlaying(): boolean {
return Boolean(this.player.isPlaying());
}
/** The latest decoded frame at the given size, or null before one exists. */
lastFrame(width: number, height: number): GrayImage | null {
const bytes = width * height;
if (bytes <= 0) return null;
if (!this.frameView || this.frameView.length !== bytes) {
this.frameBuffer = java.nio.ByteBuffer.allocateDirect(bytes);
this.frameView = new Uint8Array((ArrayBuffer as any).from(this.frameBuffer));
}
if (!this.player.copyLastFrame(this.frameBuffer)) return null;
const image = new GrayImage(width, height, 0);
image.pixels.set(this.frameView);
return image;
}
/** End playback and free the decoder and audio (the last frame stays readable). */
stop(): void {
this.player.stop();
}
release(): void {
this.player.release();
this.frameBuffer = null;
this.frameView = null;
}
}
+6
View File
@@ -44,6 +44,10 @@ export type InProcessWindowOptions = {
onSystemMenuOpened?: () => void;
onAppMenuOpened?: () => void;
setScreenOn?: ShellWindow["setScreenOn"];
/** See ShellWindow.keepsScreenOn. */
keepsScreenOn?: ShellWindow["keepsScreenOn"];
/** The window came to or left the foreground (after its surface visibility follows). */
onForegroundChanged?: (foreground: boolean) => void;
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
/**
@@ -162,6 +166,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
acceptsDirectional: true,
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
keepsScreenOn: options.keepsScreenOn,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
@@ -218,6 +223,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
onFocus: options.onFocus,
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
options.onForegroundChanged?.(foreground);
},
};
const setHeightMode = (mode: WindowHeightMode) => {
+10 -1
View File
@@ -101,6 +101,12 @@ export type ShellWindow = {
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True while the window shows something the wearer watches without
* touching anything (a playing video): the idle screen timeout waits, as
* it does for voice capture, while this window is in the foreground.
*/
keepsScreenOn?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -647,7 +653,10 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
// A foreground window that is playing video holds it off the same way.
const foreground = this.foregroundWindow();
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
this.lastInputAtMs = nowMs;
return false;
}