Add a video player, launched by selecting a video file from the Files app

This commit is contained in:
James Babcock committed 2026-09-28 14:11:04 -07:00
1 parent deaa93fead
commit cb6dddd120
13 files changed
+1784 -3

No files matched your search

@@ -0,0 +1,791 @@
package com.faceclaw.app
import android.content.Context
import android.media.AudioAttributes
import android.media.AudioFocusRequest
import android.media.AudioManager
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaExtractor
import android.media.MediaFormat
import android.media.MediaPlayer
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.util.Log
import org.json.JSONObject
import java.io.IOException
import java.nio.ByteBuffer
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantLock
import java.util.zip.CRC32
import kotlin.concurrent.withLock
/**
* Plays a video file onto the glasses. A background thread decodes the video track
* (MediaExtractor + MediaCodec), scales each due frame's luma to gray (VideoFrameScaler) and
* submits it straight into a rect of a window's compositor surface, through the same
* submitSurfaceFrame entry point app workers use, so no JS runs per frame. When wanted, the
* audio track plays through a MediaPlayer on the phone's default media route, and the video
* clock follows its position.
*
* The TS side (app/native/video-player.ts) owns the window: it paints the rest of the surface
* and repaints the video rect from [copyLastFrame] whenever it renders. Frames are submitted
* only while playing, and [pause] returns only once no further frame can be submitted, so a
* repaint right after it (the paused overlay) is never overwritten.
*
* Control methods are for the main thread; listener callbacks are posted to it.
*/
class FaceclawVideoPlayer(context: Context) {
interface Listener {
/**
* "playing", "paused" or "ended", with the media position. Also sent as "paused" when a
* seek made while paused has decoded its new frame (see [copyLastFrame]).
*/
fun onStateChanged(state: String?, positionMs: Long)
/** Playback stopped on an error (unreadable file, no decoder, ...). */
fun onError(message: String?)
}
companion object {
private const val TAG = "FaceclawVideo"
private const val CODEC_TIMEOUT_US = 10_000L
/** Longest single sleep while waiting for a frame to come due, so commands stay responsive. */
private const val MAX_WAIT_US = 20_000L
/** A frame this far behind the clock is dropped undrawn... */
private const val LATE_DROP_US = 50_000L
/** ...unless nothing has been drawn for this long, so a slow phone still shows progress. */
private const val MAX_DROP_GAP_NANOS = 250_000_000L
/**
* Decoded frames closer together than this are thinned: past ~30fps the glasses link
* could never show them, and each one still costs a scale and a composite.
*/
private const val MIN_FRAME_INTERVAL_US = 30_000L
/** Audio position is ignored this long after the clock starts (output startup latency). */
private const val AUDIO_SETTLE_NANOS = 500_000_000L
/** Clock drift from the audio position that makes the video clock jump to it. */
private const val AUDIO_RESYNC_US = 60_000L
/** How often the audio position is read (a binder call into the media server). */
private const val AUDIO_CHECK_INTERVAL_NANOS = 250_000_000L
/** A seek never lands closer than this to the end, so it always shows a frame. */
private const val SEEK_END_MARGIN_US = 500_000L
private val AUDIO_ATTRIBUTES: AudioAttributes = AudioAttributes.Builder()
.setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
.build()
private fun MediaFormat.intOr(key: String, fallback: Int): Int =
if (containsKey(key)) getInteger(key) else fallback
private fun MediaFormat.longOr(key: String, fallback: Long): Long =
if (containsKey(key)) getLong(key) else fallback
private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int {
for (i in 0 until extractor.trackCount) {
val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
if (mime.startsWith(mimePrefix)) return i
}
return -1
}
/**
* Describe a video file for the settings panel, as JSON: {width, height} in display
* orientation (rotation applied), rotation, durationMs, hasAudio, mime; or {error}.
*/
@JvmStatic
fun probe(path: String?): String {
val result = JSONObject()
val extractor = MediaExtractor()
try {
extractor.setDataSource(path ?: throw IOException("no path"))
val videoTrack = findTrack(extractor, "video/")
if (videoTrack < 0) {
result.put("error", "No video track")
return result.toString()
}
var durationUs = 0L
for (i in 0 until extractor.trackCount) {
durationUs = Math.max(durationUs, extractor.getTrackFormat(i).longOr(MediaFormat.KEY_DURATION, 0L))
}
val format = extractor.getTrackFormat(videoTrack)
val rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
val codedWidth = format.intOr(MediaFormat.KEY_WIDTH, 0)
val codedHeight = format.intOr(MediaFormat.KEY_HEIGHT, 0)
val swap = rotation == 90 || rotation == 270
result.put("width", if (swap) codedHeight else codedWidth)
result.put("height", if (swap) codedWidth else codedHeight)
result.put("rotation", rotation)
result.put("durationMs", durationUs / 1000)
result.put("hasAudio", findTrack(extractor, "audio/") >= 0)
result.put("mime", format.getString(MediaFormat.KEY_MIME) ?: "")
} catch (e: Exception) {
Log.w(TAG, "probe failed for $path", e)
result.put("error", e.message ?: e.toString())
} finally {
extractor.release()
}
return result.toString()
}
}
/** What one start() call asked for; fixed for that playback's lifetime. */
private class Params(
val path: String,
val surfaceId: String,
val x: Int,
val y: Int,
val width: Int,
val height: Int,
val monochrome: Boolean,
val gamma: Float,
val audio: Boolean,
)
private enum class Wait { DUE, INTERRUPTED, STOP }
private val appContext: Context = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
private val audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager
@Volatile private var listener: Listener? = null
private val lock = ReentrantLock()
private val wake = lock.newCondition()
// Everything below is guarded by lock.
/** Bumped by every start/stop; a playback thread whose generation is stale exits. */
private var generation = 0
private var params: Params? = null
/** The user's intent: playing or paused. */
private var playing = false
private var ended = false
/** A seek (or the initial start) is decoding toward its target; the clock waits for its frame. */
private var seeking = false
private var pendingSeekUs = -1L
private var durationUs = 0L
private var clockRunning = false
/** Media time while the clock is stopped. */
private var heldUs = 0L
private var clockBaseUs = 0L
private var clockBaseNanos = 0L
private var clockStartedNanos = 0L
private var lastAudioCheckNanos = 0L
private var audio: MediaPlayer? = null
private var hasAudioFocus = false
private var focusRequest: AudioFocusRequest? = null
/** The most recently decoded frame (params.width x params.height), shown or previewed. */
private var lastFrame: ByteArray? = null
/** Checksum of the frame last submitted, so an unchanged frame is not recomposited; -1 = none. */
private var lastSubmittedChecksum = -1L
private var submitFailures = 0
private val focusListener = AudioManager.OnAudioFocusChangeListener { change ->
when (change) {
AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> {
if (change == AudioManager.AUDIOFOCUS_LOSS) lock.withLock { hasAudioFocus = false }
// Another app (a call, a music player) took the audio: pause like any player would.
if (isPlaying()) {
pause()
notifyState(generationNow(), "paused", getPositionMs())
}
}
}
}
fun setListener(listener: Listener?) {
this.listener = listener
}
/**
* Start playing [path] from [startMs] into the (rectX, rectY, width, height) rect of
* compositor surface [surfaceId], replacing any current playback. Decoding and audio setup
* happen on the playback thread; failures arrive as [Listener.onError].
*/
fun start(
path: String,
surfaceId: String,
rectX: Int,
rectY: Int,
width: Int,
height: Int,
monochrome: Boolean,
gamma: Float,
withAudio: Boolean,
startMs: Long,
) {
require(width > 0 && height > 0) { "bad video rect ${width}x$height" }
val p = Params(path, surfaceId, rectX, rectY, width, height, monochrome, gamma, withAudio)
val gen = lock.withLock {
endPlaybackLocked()
params = p
playing = true
ended = false
seeking = true
pendingSeekUs = Math.max(0L, startMs) * 1000
heldUs = pendingSeekUs
// The old frame may not match the new settings (size, monochrome).
lastFrame = null
lastSubmittedChecksum = -1
submitFailures = 0
generation
}
val thread = Thread({ Playback(gen, p).run() }, "FaceclawVideo")
thread.priority = Thread.NORM_PRIORITY + 1
thread.start()
}
/** Pause; once this returns no further frame is submitted until [resume]. */
fun pause() {
lock.withLock {
if (!playing) return
playing = false
holdClockLocked(clockUsLocked())
wake.signalAll()
}
}
fun resume() {
lock.withLock {
if (playing || ended || params == null) return
playing = true
// A fresh frame must go out even if it matches the last one: the TS side painted over
// the surface while paused.
lastSubmittedChecksum = -1
if (!seeking) startClockLocked()
wake.signalAll()
}
}
/** Seek to [positionMs]; while paused, the new frame is decoded and announced as "paused". */
fun seekTo(positionMs: Long) {
lock.withLock {
if (params == null || ended) return
var targetUs = Math.max(0L, positionMs) * 1000
if (durationUs > 0) targetUs = Math.min(targetUs, Math.max(0L, durationUs - SEEK_END_MARGIN_US))
pendingSeekUs = targetUs
seeking = true
holdClockLocked(targetUs)
lastSubmittedChecksum = -1
wake.signalAll()
}
}
fun getPositionMs(): Long = lock.withLock { clockUsLocked() / 1000 }
fun getDurationMs(): Long = lock.withLock { durationUs / 1000 }
fun isPlaying(): Boolean = lock.withLock { playing && !ended && params != null }
/**
* Copy the most recent frame into [out], which must hold exactly the video rect's
* width*height bytes (row-major 8bpp gray). False when there is no frame of that size yet.
*/
fun copyLastFrame(out: ByteBuffer): Boolean {
lock.withLock {
val frame = lastFrame ?: return false
if (out.capacity() != frame.size) return false
out.clear()
out.put(frame)
out.clear()
return true
}
}
/** End playback and free the decoder and audio; the last frame stays readable. */
fun stop() {
lock.withLock { endPlaybackLocked() }
}
/** stop(), and drop the listener and frame. */
fun release() {
lock.withLock {
endPlaybackLocked()
lastFrame = null
}
listener = null
}
private fun generationNow(): Int = lock.withLock { generation }
private fun endPlaybackLocked() {
generation++
params = null
playing = false
seeking = false
pendingSeekUs = -1
// Keep the position readable (an error's caller resumes from it).
heldUs = clockUsLocked()
clockRunning = false
releaseAudioLocked()
wake.signalAll()
}
// ---- clock (all under lock) ----
private fun clockUsLocked(): Long =
if (!clockRunning) heldUs else clockBaseUs + (System.nanoTime() - clockBaseNanos) / 1000
private fun startClockLocked() {
if (clockRunning) return
clockRunning = true
clockBaseUs = heldUs
clockBaseNanos = System.nanoTime()
clockStartedNanos = clockBaseNanos
val player = audio ?: return
requestAudioFocusLocked()
try {
player.start()
} catch (e: IllegalStateException) {
Log.w(TAG, "audio start failed", e)
}
}
private fun holdClockLocked(atUs: Long) {
heldUs = atUs
if (!clockRunning) return
clockRunning = false
try {
audio?.let { if (it.isPlaying) it.pause() }
} catch (e: IllegalStateException) {
Log.w(TAG, "audio pause failed", e)
}
}
/**
* Keep the video clock on the audio position: output latency and seeks make the two drift,
* and a picture ahead of or behind its sound is the one sync error people notice.
*/
private fun followAudioLocked() {
val player = audio ?: return
val now = System.nanoTime()
if (!clockRunning || now - clockStartedNanos < AUDIO_SETTLE_NANOS) return
if (now - lastAudioCheckNanos < AUDIO_CHECK_INTERVAL_NANOS) return
lastAudioCheckNanos = now
val audioUs = try {
if (!player.isPlaying) return // finished (audio shorter than video) or not started
player.currentPosition * 1000L
} catch (e: IllegalStateException) {
return
}
if (Math.abs(audioUs - clockUsLocked()) > AUDIO_RESYNC_US) {
clockBaseUs = audioUs
clockBaseNanos = System.nanoTime()
}
}
private fun seekAudioLocked(targetUs: Long) {
val player = audio ?: return
try {
if (Build.VERSION.SDK_INT >= 26) {
player.seekTo(targetUs / 1000, MediaPlayer.SEEK_CLOSEST)
} else {
player.seekTo((targetUs / 1000).toInt())
}
} catch (e: IllegalStateException) {
Log.w(TAG, "audio seek failed", e)
}
}
private fun requestAudioFocusLocked() {
if (hasAudioFocus) return
val result = if (Build.VERSION.SDK_INT >= 26) {
val request = focusRequest ?: AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN)
.setAudioAttributes(AUDIO_ATTRIBUTES)
.setOnAudioFocusChangeListener(focusListener, mainHandler)
.build()
.also { focusRequest = it }
audioManager.requestAudioFocus(request)
} else {
@Suppress("DEPRECATION")
audioManager.requestAudioFocus(focusListener, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN)
}
// Play regardless: the sound was asked for, and a refusal is transient (a call ending).
hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED
}
private fun releaseAudioLocked() {
val player = audio ?: return
audio = null
try {
player.release()
} catch (e: Exception) {
Log.w(TAG, "audio release failed", e)
}
if (hasAudioFocus) {
hasAudioFocus = false
if (Build.VERSION.SDK_INT >= 26) {
focusRequest?.let { audioManager.abandonAudioFocusRequest(it) }
} else {
@Suppress("DEPRECATION")
audioManager.abandonAudioFocus(focusListener)
}
}
}
// ---- notifications ----
private fun notifyState(gen: Int, state: String, positionMs: Long) {
mainHandler.post {
if (lock.withLock { gen == generation }) listener?.onStateChanged(state, positionMs)
}
}
private fun notifyError(gen: Int, message: String) {
mainHandler.post {
if (lock.withLock { gen == generation }) listener?.onError(message)
}
}
// ---- frame output (under lock) ----
private fun submitLocked(p: Params, pixels: ByteArray, checksum: Long) {
val buffer = ByteBuffer.wrap(pixels)
// Frame id 0: per-frame FrameTimings entries at video rate would crowd every real
// input/render frame out of the timing export.
val fingerprint = "video:$generation:" + java.lang.Long.toHexString(checksum)
try {
val communicator = FaceclawBleCommunicator.getActive()
if (communicator != null) {
communicator.submitSurfaceFrame(buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
} else {
FaceclawPreviewCompositor.getActive()?.submitSurfaceFrame(
buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
}
submitFailures = 0
} catch (e: RuntimeException) {
// The window's surface went away or shrank under us (closed, relaid out): drop the frame.
if (submitFailures++ == 0) Log.w(TAG, "video frame submit failed: ${e.message}")
}
}
/**
* One start() call's decode thread. Owns the extractor and decoder; everything shared with
* the control methods goes through the player's lock.
*/
private inner class Playback(private val gen: Int, private val p: Params) {
private var extractor: MediaExtractor? = null
private var codec: MediaCodec? = null
private val info = MediaCodec.BufferInfo()
private var inputDone = false
private var outputDone = false
private var trackEndUs = 0L
/** Frames before this are decoded only on the way to a seek target. */
private var skipUntilUs = 0L
private var lastDrawnPtsUs = Long.MIN_VALUE / 2
private var lastDrawnNanos = 0L
private var rotation = 0
private var lut = ByteArray(256)
private var scaler: VideoFrameScaler? = null
private var luma = ByteArray(0)
/** The frame being built; swapped with lastFrame when shown. */
private var frame = ByteArray(p.width * p.height)
private val crc = CRC32()
fun run() {
try {
open()
loop()
} catch (t: Throwable) {
Log.w(TAG, "playback failed for ${p.path}", t)
val current = lock.withLock {
val current = gen == generation
if (current) endPlaybackLocked()
current
}
// endPlaybackLocked bumped the generation; report under the new one.
if (current) notifyError(generationNow(), t.message ?: t.toString())
} finally {
try {
codec?.stop()
} catch (ignored: Throwable) {
}
codec?.release()
extractor?.release()
}
}
private fun current(): Boolean = lock.withLock { gen == generation }
private fun open() {
val ex = MediaExtractor()
extractor = ex
ex.setDataSource(p.path)
val track = findTrack(ex, "video/")
if (track < 0) throw IOException("No video track")
ex.selectTrack(track)
val format = ex.getTrackFormat(track)
rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
trackEndUs = format.longOr(MediaFormat.KEY_DURATION, 0L)
// Most video is limited ("TV") range; only an explicit full-range tag says otherwise.
val limited = format.intOr(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) !=
MediaFormat.COLOR_RANGE_FULL
lut = VideoFrameScaler.toneLut(limited, p.gamma, p.monochrome)
val mime = format.getString(MediaFormat.KEY_MIME) ?: throw IOException("Unknown video format")
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
val decoder = try {
MediaCodec.createDecoderByType(mime)
} catch (e: Exception) {
throw IOException("No decoder for $mime")
}
codec = decoder
decoder.configure(format, null, null, 0)
decoder.start()
val player = if (p.audio && findTrack(ex, "audio/") >= 0) openAudio() else null
lock.withLock {
if (gen != generation) {
player?.release()
return
}
durationUs = trackEndUs
// Positioned by the start seek, which is the first command loop() handles.
audio = player
}
}
/** A prepared MediaPlayer for the file's audio, or null (played silent) if it won't open. */
private fun openAudio(): MediaPlayer? {
val player = MediaPlayer()
return try {
player.setAudioAttributes(AUDIO_ATTRIBUTES)
player.setDataSource(p.path)
player.prepare()
player
} catch (e: Exception) {
Log.w(TAG, "audio unavailable for ${p.path}; playing silent", e)
player.release()
null
}
}
private fun loop() {
while (true) {
// Commands: stop, seek, or wait out a pause.
var seekUs = -1L
lock.withLock {
while (true) {
if (gen != generation) return
if (pendingSeekUs >= 0) {
seekUs = pendingSeekUs
pendingSeekUs = -1
break
}
if (playing || seeking) break
wake.await()
}
}
if (seekUs >= 0) {
seek(seekUs)
continue
}
if (outputDone) {
if (awaitEnd()) return
continue
}
feedInput()
val index = codec!!.dequeueOutputBuffer(info, CODEC_TIMEOUT_US)
if (index < 0) continue // try again later / format or buffers changed
val endOfStream = (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0
if (info.size > 0) {
handleFrame(index, info.presentationTimeUs)
} else {
codec!!.releaseOutputBuffer(index, false)
}
if (endOfStream) outputDone = true
}
}
private fun seek(targetUs: Long) {
lock.withLock {
if (gen != generation) return
seekAudioLocked(targetUs)
}
extractor!!.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
codec!!.flush()
inputDone = false
outputDone = false
skipUntilUs = targetUs
lastDrawnPtsUs = Long.MIN_VALUE / 2
}
private fun feedInput() {
if (inputDone) return
val decoder = codec!!
val ex = extractor!!
while (true) {
val index = decoder.dequeueInputBuffer(0)
if (index < 0) return
val buffer = decoder.getInputBuffer(index) ?: return
val size = ex.readSampleData(buffer, 0)
if (size < 0) {
decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
inputDone = true
return
}
decoder.queueInputBuffer(index, 0, size, ex.sampleTime, 0)
ex.advance()
}
}
/** Decide a decoded frame's fate (skip, preview, drop, draw); always releases it. */
private fun handleFrame(index: Int, ptsUs: Long) {
try {
if (ptsUs < skipUntilUs) return
val landingSeek = lock.withLock { seeking }
if (landingSeek) {
landSeek(index, ptsUs)
return
}
if (ptsUs - lastDrawnPtsUs < MIN_FRAME_INTERVAL_US) return
if (waitUntilDue(ptsUs) != Wait.DUE) return
val lateUs = lock.withLock { clockUsLocked() } - ptsUs
if (lateUs > LATE_DROP_US && System.nanoTime() - lastDrawnNanos < MAX_DROP_GAP_NANOS) return
val checksum = convert(index)
present(ptsUs, checksum)
} finally {
codec!!.releaseOutputBuffer(index, false)
}
}
/**
* The first frame at or past a seek target (or the start position): drawn and the clock
* started when playing, kept as the paused preview otherwise.
*/
private fun landSeek(index: Int, ptsUs: Long) {
val checksum = convert(index)
var previewed = false
lock.withLock {
if (gen != generation || pendingSeekUs >= 0) return
seeking = false
heldUs = ptsUs
swapInFrameLocked()
if (playing) {
submitLocked(p, lastFrame!!, checksum)
lastSubmittedChecksum = checksum
startClockLocked()
} else {
previewed = true
}
}
lastDrawnPtsUs = ptsUs
lastDrawnNanos = System.nanoTime()
notifyState(gen, if (previewed) "paused" else "playing", ptsUs / 1000)
}
/** Show a due frame, unless a pause or seek got in first. */
private fun present(ptsUs: Long, checksum: Long) {
lock.withLock {
if (gen != generation || !playing || pendingSeekUs >= 0) return
swapInFrameLocked()
if (checksum != lastSubmittedChecksum) {
submitLocked(p, lastFrame!!, checksum)
lastSubmittedChecksum = checksum
}
}
lastDrawnPtsUs = ptsUs
lastDrawnNanos = System.nanoTime()
}
private fun swapInFrameLocked() {
val shown = frame
val previous = lastFrame
frame = if (previous != null && previous.size == shown.size) previous else ByteArray(shown.size)
lastFrame = shown
}
/** Block until the clock reaches [ptsUs]; paused time doesn't count. */
private fun waitUntilDue(ptsUs: Long): Wait {
lock.withLock {
while (true) {
if (gen != generation) return Wait.STOP
if (pendingSeekUs >= 0) return Wait.INTERRUPTED
if (!playing) {
wake.await()
continue
}
followAudioLocked()
val now = clockUsLocked()
if (now >= ptsUs) return Wait.DUE
wake.await(Math.min(ptsUs - now, MAX_WAIT_US), TimeUnit.MICROSECONDS)
}
}
}
/**
* The decoder is drained: hold the last frame until the clock reaches the end, then
* report "ended". False when a seek interrupted the wait (playback continues).
*/
private fun awaitEnd(): Boolean {
val endUs = Math.max(trackEndUs, lastDrawnPtsUs)
val landing = lock.withLock { seeking }
if (!landing && waitUntilDue(endUs) != Wait.DUE) return !current()
lock.withLock {
if (gen != generation) return true
if (pendingSeekUs >= 0) return false
playing = false
seeking = false
ended = true
holdClockLocked(endUs)
releaseAudioLocked()
}
notifyState(gen, "ended", endUs / 1000)
return true
}
/** Scale the output buffer's frame into [frame]; returns its checksum. */
private fun convert(index: Int): Long {
val decoder = codec!!
val image = try {
decoder.getOutputImage(index)
} catch (e: Exception) {
null
}
if (image != null) {
try {
val plane = image.planes[0]
val crop = image.cropRect
val offset = copyLuma(plane.buffer) + crop.top * plane.rowStride + crop.left * plane.pixelStride
scaleInto(crop.width(), crop.height(), offset, plane.rowStride, plane.pixelStride)
} finally {
image.close()
}
} else {
// No Image access: every YUV420 layout starts with the Y plane at the buffer start.
val buffer = decoder.getOutputBuffer(index) ?: throw IOException("Decoder returned no frame")
buffer.position(info.offset)
buffer.limit(info.offset + info.size)
val format = decoder.outputFormat
val width = format.getInteger(MediaFormat.KEY_WIDTH)
val stride = format.intOr(MediaFormat.KEY_STRIDE, width)
val left = format.intOr("crop-left", 0)
val top = format.intOr("crop-top", 0)
val right = format.intOr("crop-right", width - 1)
val bottom = format.intOr("crop-bottom", format.getInteger(MediaFormat.KEY_HEIGHT) - 1)
copyLuma(buffer)
scaleInto(right - left + 1, bottom - top + 1, top * stride + left, stride, 1)
}
crc.reset()
crc.update(frame, 0, frame.size)
return crc.value
}
/** Bulk-copy the plane (one memcpy; per-byte reads from a direct buffer are slow). Returns 0. */
private fun copyLuma(buffer: ByteBuffer): Int {
val bytes = buffer.remaining()
if (luma.size < bytes) luma = ByteArray(bytes)
buffer.get(luma, 0, bytes)
return 0
}
private fun scaleInto(width: Int, height: Int, offset: Int, rowStride: Int, pixelStride: Int) {
var s = scaler
if (s == null || s.sourceWidth != width || s.sourceHeight != height) {
s = VideoFrameScaler(width, height, rotation, p.width, p.height)
scaler = s
}
s.scale(luma, offset, rowStride, pixelStride, lut, frame)
}
}
}
+18 -2
View File
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
import { isFontFile } from "../../native/font-files";
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
import { isDecodableImageFile } from "../../native/image-files";
import { isPlayableVideoFile } from "../../native/video-player";
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
import { FileBrowserLayer } from "./file-browser";
@@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & {
installEhpkApp: (path: string) => Promise<void> | void;
/** Open a font file's previewer as its own shell window. */
openFontWindow: (title: string, path: string) => void;
/** Open a video file's player (settings panel first) as its own shell window. */
openVideoWindow: (title: string, path: string) => void;
};
/**
* The Files app's launcher-opened window: a file browser over Places
* (bookmarks and storage roots). Picking any file opens an info dialog with
* metadata plus, for viewable types (text, images), the open actions.
* metadata plus, for viewable types (text, images, video), the open actions.
*/
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
let created: InProcessWindow | null = null;
const browser = new FileBrowserLayer({
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)),
isSupportedFile: (name) =>
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
// The browser handles double-click itself (up a level), so it is not
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
onLeave: () => shell.yieldFocusToSidebar(),
@@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
},
];
}
if (isPlayableVideoFile(entry.name)) {
return [
{
label: "Play video",
onSelect: (ctx) => {
ctx.stack.pop();
options.openVideoWindow(entry.name, entry.path);
},
},
];
}
if (isFontFile(entry.name)) {
// Install copies into the app-internal fonts directory, making the face
// available in the Settings font pickers. The row relabels itself with
+13
View File
@@ -13,6 +13,7 @@ import {
FILES_SURFACE_ID,
FILES_WINDOW_ID,
} from "./files-app";
import { createVideoPlayerWindow } from "./video-window";
// Document windows are closeable and non-singleton; the serial keeps their
// windowIds unique.
@@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v
});
}
function openVideoWindow(ctx: AppContext, title: string, path: string): void {
const windowId = `files:doc:${nextDocumentSerial++}`;
void ctx
.launchInProcessApp(windowId, `window:${windowId}`, (options) =>
createVideoPlayerWindow(windowId, title, path, options),
)
.catch((error) => {
ctx.appendLog(`video window failed: ${error}`);
});
}
const filesApp: AppDefinition = {
appId: "files",
title: "Files",
@@ -78,6 +90,7 @@ const filesApp: AppDefinition = {
}
},
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
}),
),
// A document arriving via Android's Share intent opens as its own window.
+86
View File
@@ -0,0 +1,86 @@
/**
* Where a video goes in the player window, for each size choice. Pure
* geometry, so it is unit-tested without the NativeScript runtime.
*/
export type VideoSize = "small" | "medium" | "large" | "full";
export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"];
export type Size = { width: number; height: number };
export type Rect = Size & { x: number; y: number };
/**
* Bounding boxes for the fixed sizes; a video is fitted inside, keeping its
* aspect. Large and full fit the whole viewport instead (full in a tall
* window). Bigger pictures cost the glasses link more per frame, so smaller
* sizes play at a higher frame rate.
*/
const FIXED_BOXES: Partial<Record<VideoSize, Size>> = {
small: { width: 160, height: 120 },
medium: { width: 240, height: 180 },
};
export function videoSizeLabel(size: VideoSize): string {
switch (size) {
case "small":
return "Small";
case "medium":
return "Medium";
case "large":
return "Large";
case "full":
return "Full screen";
}
}
/** Full screen needs the tall window band; the rest play in the standard one. */
export function videoHeightMode(size: VideoSize): "min" | "max" {
return size === "full" ? "max" : "min";
}
/**
* The largest size with the video's aspect ratio that fits in box, with even
* dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel
* columns) of at least 2. Upscales small videos too: the size is the
* viewer's choice.
*/
export function fitVideo(video: Size, box: Size): Size {
if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) };
const scale = Math.min(box.width / video.width, box.height / video.height);
return {
width: evenFloor(Math.min(box.width, Math.round(video.width * scale))),
height: evenFloor(Math.min(box.height, Math.round(video.height * scale))),
};
}
/**
* The video's rect within a viewport, centered. x snaps to a multiple of 4
* and y to a multiple of 2 so frame deltas don't straddle an extra encoder
* block at each edge.
*/
export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect {
const fixed = FIXED_BOXES[size];
const box = fixed
? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) }
: viewport;
const fitted = fitVideo(video, box);
return {
x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4),
y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2),
...fitted,
};
}
/** "3:52", or "1:02:03" past an hour. */
export function formatMediaTime(ms: number): string {
const total = Math.max(0, Math.floor(ms / 1000));
const hours = Math.floor(total / 3600);
const minutes = Math.floor((total % 3600) / 60);
const seconds = String(total % 60).padStart(2, "0");
return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`;
}
function evenFloor(value: number): number {
return Math.max(2, Math.floor(value / 2) * 2);
}
+443
View File
@@ -0,0 +1,443 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { PHOTO_GAMMA } from "../../native/image-files";
import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player";
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
import { type InputEvent } from "../../ui/gestures";
import { type Layer, type LayerContext } from "../../ui/layers";
import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu";
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { appViewportSize } from "../../ui/shell/geometry";
import {
formatMediaTime,
VIDEO_SIZES,
videoHeightMode,
videoRect,
videoSizeLabel,
type Rect,
type VideoSize,
} from "./video-layout";
export const videoSizeSetting = new ConfigSettingEnum<VideoSize>({
id: "video-size",
label: "Size",
storageKey: "video.size",
defaultValue: "large",
values: VIDEO_SIZES,
formatValue: videoSizeLabel,
});
export const videoAudioSetting = new ConfigSettingBoolean({
id: "video-audio",
label: "Audio",
storageKey: "video.audio",
defaultValue: true,
description: "Play the video's sound on the phone's current audio output.",
});
export const videoMonochromeSetting = new ConfigSettingBoolean({
id: "video-monochrome",
label: "Monochrome",
storageKey: "video.monochrome",
defaultValue: false,
description: "Show only black and white: far less to send per frame, so a higher frame rate.",
});
/** How far one ring scroll seeks. */
const SEEK_STEP_MS = 10_000;
const MARGIN_X = 18;
const TITLE_Y = 10;
const PANEL_WIDTH = 340;
/**
* One opened video: its probe result, the native player, and the window
* glue. The window's base layer is the settings panel; playing pushes the
* playback layer, and backing out of it (or the video ending) returns to the
* panel with the position kept for Resume.
*/
class VideoSession {
readonly probe: VideoInfo | { error: string };
private player: VideoPlayer | null = null;
private window: InProcessWindow | null = null;
private playback: VideoPlaybackLayer | null = null;
/** Where Resume starts; 0 = from the beginning. */
resumeMs = 0;
/** The last playback error, shown on the panel until the next Play. */
error: string | null = null;
constructor(
readonly path: string,
readonly title: string,
readonly surfaceId: string,
) {
this.probe = probeVideo(path);
}
get info(): VideoInfo | null {
return "error" in this.probe ? null : this.probe;
}
attach(window: InProcessWindow): void {
this.window = window;
}
isPlaying(): boolean {
return this.player?.isPlaying() ?? false;
}
positionMs(): number {
return this.player?.positionMs() ?? this.resumeMs;
}
/** Current rect for a size choice, in the viewport of the band that size plays in. */
rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect {
const info = this.info;
return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport);
}
play(ctx: LayerContext, fromStart: boolean): void {
const window = this.window;
const info = this.info;
if (!window || !info) return;
this.error = null;
const size = videoSizeSetting.get();
window.setHeightMode(videoHeightMode(size));
const viewport = ctx.stack.getBaseSize();
const rect = this.rectFor(size, viewport);
const player = this.ensurePlayer();
this.playback = new VideoPlaybackLayer(this, rect, viewport);
ctx.stack.push(this.playback);
player.start({
path: this.path,
surfaceId: this.surfaceId,
rect,
monochrome: videoMonochromeSetting.get(),
gamma: PHOTO_GAMMA,
audio: info.hasAudio && videoAudioSetting.get(),
startMs: fromStart ? 0 : this.resumeMs,
});
}
togglePause(): void {
const player = this.player;
if (!player) return;
if (player.isPlaying()) player.pause();
else player.resume();
}
seekBy(deltaMs: number): void {
const player = this.player;
if (!player) return;
const duration = this.info?.durationMs ?? 0;
const target = Math.max(0, player.positionMs() + deltaMs);
player.seekTo(duration > 0 ? Math.min(target, duration) : target);
}
/** The window went to the background or the screen went off: stop, keep the place. */
pauseForHost(): void {
if (!this.player?.isPlaying()) return;
this.player.pause();
// Paint the paused overlay now, so the retained surface shows it on return.
this.window?.requestRender();
}
/** Leave the playback layer for the panel, remembering where to resume. */
returnToPanel(resumeMs: number): void {
const player = this.player;
if (player) {
player.pause();
player.stop();
}
this.resumeMs = resumeMs;
const window = this.window;
const playback = this.playback;
this.playback = null;
if (window && playback) window.stack.popThrough(playback);
window?.setHeightMode("min");
window?.requestRender();
}
lastFrame(rect: Rect): GrayImage | null {
return this.player?.lastFrame(rect.width, rect.height) ?? null;
}
release(): void {
this.player?.release();
this.player = null;
}
private ensurePlayer(): VideoPlayer {
if (!this.player) {
this.player = new VideoPlayer({
onState: (state, positionMs) => this.onPlayerState(state, positionMs),
onError: (message) => {
this.error = message;
this.returnToPanel(this.positionMs());
},
});
}
return this.player;
}
private onPlayerState(state: VideoPlaybackState, _positionMs: number): void {
if (state === "ended") {
this.returnToPanel(0);
return;
}
// A paused seek decoded its new frame: repaint it under the overlay.
if (state === "paused") this.window?.requestRender();
}
}
type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean };
/**
* The pre-play settings panel (the window's root): picture size, audio and
* monochrome, all remembered for the next video, then Play (or Resume plus
* Play from start once a position is kept).
*/
class VideoSettingsLayer implements Layer {
private readonly menu = new Menu<PanelRow>({
wrap: false,
rowGap: 1,
highlight: { radius: 6 },
getHeight: () => listRowHeight(getDefaultSmallFont()),
draw: (args) => this.drawRow(args),
});
constructor(private readonly session: VideoSession) {}
private rows(): PanelRow[] {
if (!this.session.info) return [];
const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }];
rows.push({ kind: "play", fromStart: false });
if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true });
return rows;
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220);
const step = lineStep(font);
let y = TITLE_Y + step;
const info = this.session.info;
const errors: string[] = [];
if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`);
if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`);
if (info) {
const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"];
image.drawText(font, MARGIN_X, y, facts.join(" · "), 130);
y += step;
}
for (const error of errors) {
for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) {
image.drawText(font, MARGIN_X, y, line, 200);
y += step;
}
}
y += 6;
const rows = this.rows();
const selected = this.menu.selectedItem;
const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1;
// Land on Resume/Play by default: the settings usually stay as they were.
const playIndex = rows.findIndex((row) => row.kind === "play");
this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined);
const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6));
this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) },
ctx.stack.isFocused());
return image;
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
switch (event.type) {
case "scroll-up":
case "scroll-down":
await this.menu.handleInput(event);
return;
case "click": {
const row = this.menu.selectedItem;
if (row) this.activate(row, ctx);
return;
}
default:
return;
}
}
private activate(row: PanelRow, ctx: LayerContext): void {
switch (row.kind) {
case "size":
videoSizeSetting.set(videoSizeSetting.next());
return;
case "audio":
if (this.session.info?.hasAudio) videoAudioSetting.toggle();
return;
case "monochrome":
videoMonochromeSetting.toggle();
return;
case "play":
this.session.play(ctx, row.fromStart);
return;
}
}
private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs<PanelRow>): void {
const font = getDefaultSmallFont();
const textX = x + 10;
const textWidth = width - 20;
switch (item.kind) {
case "size": {
const size = videoSizeSetting.get();
// The picture size this choice gets in its own band (full screen plays in the tall window).
const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size)));
drawRightValueMenuItem(image, font, textX, y, textWidth, "Size",
`${videoSizeLabel(size)} ${rect.width}x${rect.height}`);
return;
}
case "audio": {
const hasAudio = this.session.info?.hasAudio ?? false;
drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)",
hasAudio && videoAudioSetting.get(), selected);
return;
}
case "monochrome":
drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected);
return;
case "play": {
const resumeMs = this.session.resumeMs;
const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play";
image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200);
return;
}
}
}
}
function sameRow(a: PanelRow, b: PanelRow): boolean {
return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart);
}
/**
* The playing video. The native player draws frames into the video rect on
* its own; this layer paints only when the window renders (input, pause,
* focus): black around the latest frame, plus a status box while paused.
* Click pauses/resumes, scroll seeks 10s, double-click returns to the panel.
*/
class VideoPlaybackLayer implements Layer {
private leaving = false;
constructor(
private readonly session: VideoSession,
private readonly rect: Rect,
/** The viewport the rect was laid out in. */
private readonly viewport: { width: number; height: number },
) {}
paint(ctx: LayerContext): GrayImage {
const { width, height } = ctx.stack.getBaseSize();
if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) {
// The window was relaid out (display mode change) and the native player's rect no
// longer fits the surface: back out to the panel, which lays out afresh on Play.
this.leaving = true;
setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0);
}
const image = new GrayImage(width, height, 0);
const frame = this.session.lastFrame(this.rect);
if (frame) image.bitBlt(frame, this.rect.x, this.rect.y);
if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null);
return image;
}
async handleInput(event: InputEvent): Promise<void> {
switch (event.type) {
case "click":
this.session.togglePause();
return;
case "scroll-up":
this.session.seekBy(-SEEK_STEP_MS);
return;
case "scroll-down":
this.session.seekBy(SEEK_STEP_MS);
return;
case "double-click":
this.session.returnToPanel(this.session.positionMs());
return;
default:
return;
}
}
/**
* "Paused 1:23 / 3:52" over a progress bar, below the picture when there is
* room (small sizes), else over its bottom edge.
*/
private drawPausedBox(image: GrayImage, loading: boolean): void {
const font = getDefaultSmallFont();
const duration = this.session.info?.durationMs ?? 0;
const position = this.session.positionMs();
const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`;
const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24);
const boxHeight = font.lineHeight + 14;
const rect = this.rect;
const x = Math.round(rect.x + (rect.width - boxWidth) / 2);
const below = rect.y + rect.height + 4;
const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6);
image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6);
image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6);
image.drawText(font, x + 12, y + 5, text, 220);
if (!loading && duration > 0) {
const barWidth = boxWidth - 24;
image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40);
image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200);
}
}
}
/** A video file's player as its own window, opened from the Files app. */
export function createVideoPlayerWindow(
windowId: string,
title: string,
path: string,
options: InProcessAppOptions,
): InProcessWindow {
const session = new VideoSession(path, title, `window:${windowId}`);
const window = createInProcessWindow({
appId: "files",
windowId,
title,
iconLetter: "V",
icon: "film",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)),
// A playing video is the thing being watched: no idle screen-off.
keepsScreenOn: () => session.isPlaying(),
onForegroundChanged: (foreground) => {
if (!foreground) session.pauseForHost();
},
setScreenOn: (on) => {
if (!on) session.pauseForHost();
},
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
reconfigureSurface: options.reconfigureSurface,
onClosed: () => {
session.release();
options.onClosed();
},
});
session.attach(window);
return window;
}
+148
View File
@@ -0,0 +1,148 @@
/**
* Bridge to the Android FaceclawVideoPlayer: decodes a video file and
* streams its frames, scaled to gray, straight into a rect of a window's
* compositor surface, with the audio on the phone's default media route.
* Android only; on iOS probeVideo reports an error and nothing plays.
*/
import { Utils } from "@nativescript/core";
import { GrayImage } from "../graphics/image";
declare const com: any;
declare const java: any;
declare const global: any;
/** Containers MediaExtractor opens (AVI is not one of them). */
const PLAYABLE_VIDEO = /\.(mp4|m4v|mkv|webm|3gp|mov)$/i;
export function isPlayableVideoFile(name: string): boolean {
return PLAYABLE_VIDEO.test(name);
}
export type VideoInfo = {
/** Display size, rotation already applied. */
width: number;
height: number;
durationMs: number;
hasAudio: boolean;
};
/** Read a video's size, duration and tracks; an error string when it can't be played. */
export function probeVideo(path: string): VideoInfo | { error: string } {
if (!global.isAndroid) return { error: "Video playback is not supported on this phone" };
try {
const raw = JSON.parse(String(com.faceclaw.app.FaceclawVideoPlayer.probe(path)));
if (raw.error) return { error: String(raw.error) };
return {
width: Number(raw.width) || 0,
height: Number(raw.height) || 0,
durationMs: Number(raw.durationMs) || 0,
hasAudio: Boolean(raw.hasAudio),
};
} catch (error) {
return { error: String(error) };
}
}
export type VideoPlaybackState = "playing" | "paused" | "ended";
export type VideoPlayerCallbacks = {
/** State change, or a paused seek that landed on a new frame (state "paused"). */
onState: (state: VideoPlaybackState, positionMs: number) => void;
onError: (message: string) => void;
};
export type VideoStartOptions = {
path: string;
surfaceId: string;
/** The video's rect in surface coordinates. */
rect: { x: number; y: number; width: number; height: number };
monochrome: boolean;
/** Tone curve for gray output (ignored for monochrome); see PHOTO_GAMMA. */
gamma: number;
audio: boolean;
startMs: number;
};
/** One player; frames go to whatever surface the latest start() named. Main thread only. */
export class VideoPlayer {
private readonly player: any;
// The Java listener proxy must stay referenced or it gets GC'd.
private readonly listenerProxy: any;
// Reused direct buffer (and its JS view) for copyLastFrame; see java-direct-buffer.ts
// for why frames cross as a Java-allocated buffer.
private frameBuffer: any = null;
private frameView: Uint8Array | null = null;
constructor(callbacks: VideoPlayerCallbacks) {
this.player = new com.faceclaw.app.FaceclawVideoPlayer(Utils.android.getApplicationContext());
this.listenerProxy = new com.faceclaw.app.FaceclawVideoPlayer.Listener({
onStateChanged: (state: string, positionMs: number) => {
callbacks.onState(String(state) as VideoPlaybackState, Number(positionMs));
},
onError: (message: string) => callbacks.onError(String(message ?? "Playback failed")),
});
this.player.setListener(this.listenerProxy);
}
start(options: VideoStartOptions): void {
const { rect } = options;
this.player.start(
options.path,
options.surfaceId,
Math.round(rect.x),
Math.round(rect.y),
Math.round(rect.width),
Math.round(rect.height),
options.monochrome,
options.gamma,
options.audio,
Math.max(0, Math.round(options.startMs)),
);
}
/** Once this returns, no more frames reach the surface until resume(). */
pause(): void {
this.player.pause();
}
resume(): void {
this.player.resume();
}
seekTo(positionMs: number): void {
this.player.seekTo(Math.max(0, Math.round(positionMs)));
}
positionMs(): number {
return Number(this.player.getPositionMs());
}
isPlaying(): boolean {
return Boolean(this.player.isPlaying());
}
/** The latest decoded frame at the given size, or null before one exists. */
lastFrame(width: number, height: number): GrayImage | null {
const bytes = width * height;
if (bytes <= 0) return null;
if (!this.frameView || this.frameView.length !== bytes) {
this.frameBuffer = java.nio.ByteBuffer.allocateDirect(bytes);
this.frameView = new Uint8Array((ArrayBuffer as any).from(this.frameBuffer));
}
if (!this.player.copyLastFrame(this.frameBuffer)) return null;
const image = new GrayImage(width, height, 0);
image.pixels.set(this.frameView);
return image;
}
/** End playback and free the decoder and audio (the last frame stays readable). */
stop(): void {
this.player.stop();
}
release(): void {
this.player.release();
this.frameBuffer = null;
this.frameView = null;
}
}
+6
View File
@@ -44,6 +44,10 @@ export type InProcessWindowOptions = {
onSystemMenuOpened?: () => void;
onAppMenuOpened?: () => void;
setScreenOn?: ShellWindow["setScreenOn"];
/** See ShellWindow.keepsScreenOn. */
keepsScreenOn?: ShellWindow["keepsScreenOn"];
/** The window came to or left the foreground (after its surface visibility follows). */
onForegroundChanged?: (foreground: boolean) => void;
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
/**
@@ -162,6 +166,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
acceptsDirectional: true,
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
keepsScreenOn: options.keepsScreenOn,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
@@ -218,6 +223,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
onFocus: options.onFocus,
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
options.onForegroundChanged?.(foreground);
},
};
const setHeightMode = (mode: WindowHeightMode) => {
+10 -1
View File
@@ -101,6 +101,12 @@ export type ShellWindow = {
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True while the window shows something the wearer watches without
* touching anything (a playing video): the idle screen timeout waits, as
* it does for voice capture, while this window is in the foreground.
*/
keepsScreenOn?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -647,7 +653,10 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
// A foreground window that is playing video holds it off the same way.
const foreground = this.foregroundWindow();
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -0,0 +1,133 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/**
* Scales one decoded video frame's luma plane into an 8bpp gray rect for a compositor surface:
* a box filter (averaging up to [MAX_TAPS] x [MAX_TAPS] evenly spaced samples per output pixel,
* nearest-neighbor when upscaling), then the container's display rotation, then a tone lookup.
*
* Only luma is read: the glasses show gray, and Y is exactly the gray a decoder hands back
* without any color conversion. Platform code owns decoding and copies the plane into a
* ByteArray; this class owns the pixel math. One instance serves one (source size, rotation,
* output size) combination and reuses its scratch buffers across frames; not thread-safe.
*/
class VideoFrameScaler(
/** Visible luma size in coded (unrotated) orientation. */
val sourceWidth: Int,
val sourceHeight: Int,
/** Clockwise display rotation in degrees: 0, 90, 180 or 270. */
val rotation: Int,
/** Output size in display orientation. */
val outputWidth: Int,
val outputHeight: Int,
) {
companion object {
/** Most samples averaged along each axis per output pixel; bounds work for big downscales. */
const val MAX_TAPS = 4
/** Snap any angle to the nearest of 0/90/180/270, the only rotations containers use. */
@JvmStatic
fun normalizeRotation(degrees: Int): Int = (((((degrees % 360) + 360) % 360 + 45) / 90) % 4) * 90
/**
* Luma to display gray. Limited ("TV") range video codes black at 16 and white at 235, so
* it is stretched to 0..255 first. Monochrome thresholds at mid-gray to 0 or 255 (a
* 1-bit image compresses and delta-encodes far better than 16 levels); otherwise the
* gamma curve applies, as for photos (see GrayPacket.fromArgb).
*/
@JvmStatic
fun toneLut(limitedRange: Boolean, gamma: Float, monochrome: Boolean): ByteArray {
val curve = GrayPacket.toneCurve(gamma)
val lut = ByteArray(256)
for (y in 0 until 256) {
val full = if (limitedRange) ((y - 16) * 255 + 109) / 219 else y
val v = full.coerceIn(0, 255)
lut[y] = (if (monochrome) (if (v >= 128) 255 else 0) else curve[v]).toByte()
}
return lut
}
/**
* Sample positions for each output index along one axis: [count] entries per output,
* evenly spaced inside that output's box of source pixels.
*/
private fun taps(source: Int, output: Int, count: Int): IntArray {
val positions = IntArray(output * count)
for (i in 0 until output) {
for (j in 0 until count) {
// Center of sub-box j of output i, in source pixels: (i + (j + 0.5)/count) * source / output.
val numerator = (i.toLong() * count + j) * 2 + 1
val position = (numerator * source / (2L * count * output)).toInt()
positions[i * count + j] = position.coerceIn(0, source - 1)
}
}
return positions
}
}
private val swapsAxes = rotation == 90 || rotation == 270
/** Scaled size before rotation (coded orientation). */
private val scaledWidth = if (swapsAxes) outputHeight else outputWidth
private val scaledHeight = if (swapsAxes) outputWidth else outputHeight
private val tapsX = ((sourceWidth + scaledWidth - 1) / scaledWidth).coerceIn(1, MAX_TAPS)
private val tapsY = ((sourceHeight + scaledHeight - 1) / scaledHeight).coerceIn(1, MAX_TAPS)
private val xs = taps(sourceWidth, scaledWidth, tapsX)
private val ys = taps(sourceHeight, scaledHeight, tapsY)
private val sums = IntArray(scaledWidth)
/** Unrotated scaled frame, needed only when a rotation follows. */
private val scaled: ByteArray? = if (rotation == 0) null else ByteArray(scaledWidth * scaledHeight)
init {
require(sourceWidth > 0 && sourceHeight > 0) { "bad source size ${sourceWidth}x$sourceHeight" }
require(outputWidth > 0 && outputHeight > 0) { "bad output size ${outputWidth}x$outputHeight" }
require(rotation == 0 || rotation == 90 || rotation == 180 || rotation == 270) { "bad rotation $rotation" }
}
/**
* Scale the luma plane into [out] (outputWidth * outputHeight bytes, row-major). The visible
* pixel (x, y) is at luma[offset + y * rowStride + x * pixelStride]; [lut] is a 256-entry
* table from [toneLut].
*/
fun scale(luma: ByteArray, offset: Int, rowStride: Int, pixelStride: Int, lut: ByteArray, out: ByteArray) {
require(out.size >= outputWidth * outputHeight) { "output buffer too small" }
val target = scaled ?: out
val count = tapsX * tapsY
val half = count / 2
for (oy in 0 until scaledHeight) {
sums.fill(0)
for (ty in 0 until tapsY) {
val row = offset + ys[oy * tapsY + ty] * rowStride
var tap = 0
for (ox in 0 until scaledWidth) {
var sum = 0
for (tx in 0 until tapsX) {
sum += luma[row + xs[tap++] * pixelStride].toInt() and 0xff
}
sums[ox] += sum
}
}
val base = oy * scaledWidth
for (ox in 0 until scaledWidth) {
target[base + ox] = lut[(sums[ox] + half) / count]
}
}
if (scaled != null) rotate(scaled, out)
}
/** Rotate the coded-orientation frame clockwise by [rotation] into display orientation. */
private fun rotate(source: ByteArray, out: ByteArray) {
val w = scaledWidth
val h = scaledHeight
for (y in 0 until outputHeight) {
val base = y * outputWidth
for (x in 0 until outputWidth) {
out[base + x] = when (rotation) {
90 -> source[(h - 1 - x) * w + y]
180 -> source[(h - 1 - y) * w + (w - 1 - x)]
else -> source[x * w + (w - 1 - y)]
}
}
}
}
}
+3
View File
@@ -56,6 +56,7 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
const load = loader({ global: { isIOS: true } }, {
'../../native/file-access': {}, '../../native/font-files': { isFontFile: () => false },
'../../graphics/installed-fonts': {}, '../../native/image-files': { isDecodableImageFile: () => false },
'../../native/video-player': { isPlayableVideoFile: () => true },
'../evenhub/installed-apps': { readEvenHubPackageManifest: () => null }, '../evenhub/permission-dialog': {},
'./file-browser': { FileBrowserLayer: class { constructor(options) { browserOptions = options; } } },
'./file-info-dialog': { FileInfoDialogLayer: class { constructor(entry, value) { actions = value; } } },
@@ -64,6 +65,8 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
});
load('app/apps/files/files-app.ts').createFilesAppWindow({ openEhpkApp: path => { opened = path; }, installEhpkApp: path => { installed = path; } });
assert.equal(browserOptions.isSupportedFile('test.EHPK'), true);
// Video playback is Android-only for now.
assert.equal(browserOptions.isSupportedFile('clip.mp4'), false);
const ctx = { stack: { push() {}, pop() {} } };
browserOptions.onFilePicked({ name: 'test.ehpk', path: '/Documents/test.ehpk' }, ctx);
assert.deepEqual(Array.from(actions, a => a.label), ['Run app', 'Install']);
@@ -0,0 +1,75 @@
package com.faceclaw.app
import kotlin.test.*
class VideoFrameScalerTest {
private val identity = ByteArray(256) { it.toByte() }
private fun scale(scaler: VideoFrameScaler, luma: ByteArray, lut: ByteArray = identity): IntArray {
val out = ByteArray(scaler.outputWidth * scaler.outputHeight)
scaler.scale(luma, 0, scaler.sourceWidth, 1, lut, out)
return IntArray(out.size) { out[it].toInt() and 0xff }
}
@Test fun twoToOneDownscaleAveragesEachBox() {
// 4x2 source -> 2x1: each output is the mean of a 2x2 box.
val luma = byteArrayOf(0, 100, 200.toByte(), 200.toByte(), 20, 80, 100, 100)
val scaler = VideoFrameScaler(4, 2, 0, 2, 1)
assertContentEquals(intArrayOf(50, 150), scale(scaler, luma))
}
@Test fun largeDownscaleStaysWithinTheTapBudget() {
// 100:1 horizontally still averages only MAX_TAPS samples, spread across the box.
val luma = ByteArray(200) { (if (it % 100 < 50) 0 else 200).toByte() }
val scaler = VideoFrameScaler(200, 1, 0, 2, 1)
assertContentEquals(intArrayOf(100, 100), scale(scaler, luma))
}
@Test fun upscaleIsNearestNeighbor() {
val scaler = VideoFrameScaler(2, 1, 0, 4, 2)
assertContentEquals(intArrayOf(10, 10, 90, 90, 10, 10, 90, 90), scale(scaler, byteArrayOf(10, 90)))
}
@Test fun offsetAndStridesSelectTheVisiblePlane() {
// A 2x2 visible crop at (1, 1) of a 4-wide plane, with interleaved samples (pixelStride 2).
val plane = ByteArray(40)
fun put(x: Int, y: Int, v: Int) { plane[(1 + y) * 10 + 1 + x * 2] = v.toByte() }
put(0, 0, 11); put(1, 0, 22); put(0, 1, 33); put(1, 1, 44)
val scaler = VideoFrameScaler(2, 2, 0, 2, 2)
val out = ByteArray(4)
scaler.scale(plane, 11, 10, 2, identity, out)
assertContentEquals(byteArrayOf(11, 22, 33, 44), out)
}
@Test fun rotationsMatchTheContainerClockwiseConvention() {
// Coded 3x2: 1 2 3
// 4 5 6
val luma = byteArrayOf(1, 2, 3, 4, 5, 6)
assertContentEquals(intArrayOf(4, 1, 5, 2, 6, 3), scale(VideoFrameScaler(3, 2, 90, 2, 3), luma))
assertContentEquals(intArrayOf(6, 5, 4, 3, 2, 1), scale(VideoFrameScaler(3, 2, 180, 3, 2), luma))
assertContentEquals(intArrayOf(3, 6, 2, 5, 1, 4), scale(VideoFrameScaler(3, 2, 270, 2, 3), luma))
}
@Test fun normalizeRotationSnapsToQuarterTurns() {
assertEquals(0, VideoFrameScaler.normalizeRotation(0))
assertEquals(90, VideoFrameScaler.normalizeRotation(90))
assertEquals(270, VideoFrameScaler.normalizeRotation(-90))
assertEquals(180, VideoFrameScaler.normalizeRotation(540))
assertEquals(0, VideoFrameScaler.normalizeRotation(359))
}
@Test fun limitedRangeStretchesAndMonochromeThresholds() {
val gray = VideoFrameScaler.toneLut(limitedRange = true, gamma = 1f, monochrome = false)
assertEquals(0, gray[16].toInt() and 0xff)
assertEquals(0, gray[0].toInt() and 0xff)
assertEquals(255, gray[235].toInt() and 0xff)
assertEquals(255, gray[255].toInt() and 0xff)
val mono = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = true)
assertEquals(0, mono[127].toInt() and 0xff)
assertEquals(255, mono[128].toInt() and 0xff)
// Gamma darkens midtones but leaves the ends alone.
val photo = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = false)
assertTrue((photo[128].toInt() and 0xff) < 64)
assertEquals(255, photo[255].toInt() and 0xff)
}
}
+1
View File
@@ -36,6 +36,7 @@
"../app/g2/glasses-display-state.ts",
"../app/g2/glance-state.ts",
"../app/apps/glanceboard/layout.ts",
"../app/apps/files/video-layout.ts",
"../app/g2/pairing-candidates.ts",
"../app/apps/calculator/math/*.ts",
"../app/apps/microphones/mic-protocol.ts",
+57
View File
@@ -0,0 +1,57 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const {
fitVideo,
formatMediaTime,
videoHeightMode,
videoRect,
} = require("../.test-build/app/apps/files/video-layout.js");
const WINDOW = { width: 576, height: 260 };
const TALL_WINDOW = { width: 576, height: 452 };
// ../g2flash/demos/badapple.mp4
const BAD_APPLE = { width: 1444, height: 1080 };
test("fixed sizes fit their box and keep the aspect ratio", () => {
assert.deepEqual(videoRect("small", BAD_APPLE, WINDOW), { x: 208, y: 70, width: 160, height: 120 });
assert.deepEqual(videoRect("medium", BAD_APPLE, WINDOW), { x: 168, y: 40, width: 240, height: 180 });
const wide = videoRect("small", { width: 1920, height: 1080 }, WINDOW);
assert.equal(wide.width, 160);
assert.equal(wide.height, 90);
});
test("large fills the window height; full screen fills the tall window's width", () => {
assert.deepEqual(videoRect("large", BAD_APPLE, WINDOW), { x: 112, y: 0, width: 348, height: 260 });
assert.deepEqual(videoRect("full", BAD_APPLE, TALL_WINDOW), { x: 0, y: 10, width: 576, height: 430 });
assert.equal(videoHeightMode("full"), "max");
assert.equal(videoHeightMode("large"), "min");
});
test("rects are aligned for the delta encoder and stay inside the viewport", () => {
for (const size of ["small", "medium", "large", "full"]) {
for (const video of [BAD_APPLE, { width: 1080, height: 1920 }, { width: 100, height: 20 }, { width: 7, height: 5 }]) {
for (const viewport of [WINDOW, TALL_WINDOW, { width: 150, height: 100 }]) {
const rect = videoRect(size, video, viewport);
const where = `${size} ${video.width}x${video.height} in ${viewport.width}x${viewport.height}`;
assert.equal(rect.x % 4, 0, where);
assert.equal(rect.y % 2, 0, where);
assert.equal(rect.width % 2, 0, where);
assert.equal(rect.height % 2, 0, where);
assert.ok(rect.width >= 2 && rect.height >= 2, where);
assert.ok(rect.x + rect.width <= viewport.width, where);
assert.ok(rect.y + rect.height <= viewport.height, where);
}
}
}
});
test("a tiny video is scaled up to the chosen size", () => {
assert.deepEqual(fitVideo({ width: 80, height: 60 }, { width: 240, height: 180 }), { width: 240, height: 180 });
});
test("media times", () => {
assert.equal(formatMediaTime(0), "0:00");
assert.equal(formatMediaTime(232_130), "3:52");
assert.equal(formatMediaTime(3_723_000), "1:02:03");
});