forked from Blendman974/faceclaw
Add a video player, launched by selecting a video file from the Files app
This commit is contained in:
1 parent
deaa93fead
commit
cb6dddd120
13 files changed
+1784
-3
No files matched your search
@@ -0,0 +1,791 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.media.AudioAttributes
|
||||||
|
import android.media.AudioFocusRequest
|
||||||
|
import android.media.AudioManager
|
||||||
|
import android.media.MediaCodec
|
||||||
|
import android.media.MediaCodecInfo
|
||||||
|
import android.media.MediaExtractor
|
||||||
|
import android.media.MediaFormat
|
||||||
|
import android.media.MediaPlayer
|
||||||
|
import android.os.Build
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.Looper
|
||||||
|
import android.util.Log
|
||||||
|
|
||||||
|
import org.json.JSONObject
|
||||||
|
|
||||||
|
import java.io.IOException
|
||||||
|
import java.nio.ByteBuffer
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import java.util.concurrent.locks.ReentrantLock
|
||||||
|
import java.util.zip.CRC32
|
||||||
|
import kotlin.concurrent.withLock
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Plays a video file onto the glasses. A background thread decodes the video track
|
||||||
|
* (MediaExtractor + MediaCodec), scales each due frame's luma to gray (VideoFrameScaler) and
|
||||||
|
* submits it straight into a rect of a window's compositor surface, through the same
|
||||||
|
* submitSurfaceFrame entry point app workers use, so no JS runs per frame. When wanted, the
|
||||||
|
* audio track plays through a MediaPlayer on the phone's default media route, and the video
|
||||||
|
* clock follows its position.
|
||||||
|
*
|
||||||
|
* The TS side (app/native/video-player.ts) owns the window: it paints the rest of the surface
|
||||||
|
* and repaints the video rect from [copyLastFrame] whenever it renders. Frames are submitted
|
||||||
|
* only while playing, and [pause] returns only once no further frame can be submitted, so a
|
||||||
|
* repaint right after it (the paused overlay) is never overwritten.
|
||||||
|
*
|
||||||
|
* Control methods are for the main thread; listener callbacks are posted to it.
|
||||||
|
*/
|
||||||
|
class FaceclawVideoPlayer(context: Context) {
|
||||||
|
interface Listener {
|
||||||
|
/**
|
||||||
|
* "playing", "paused" or "ended", with the media position. Also sent as "paused" when a
|
||||||
|
* seek made while paused has decoded its new frame (see [copyLastFrame]).
|
||||||
|
*/
|
||||||
|
fun onStateChanged(state: String?, positionMs: Long)
|
||||||
|
|
||||||
|
/** Playback stopped on an error (unreadable file, no decoder, ...). */
|
||||||
|
fun onError(message: String?)
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val TAG = "FaceclawVideo"
|
||||||
|
private const val CODEC_TIMEOUT_US = 10_000L
|
||||||
|
/** Longest single sleep while waiting for a frame to come due, so commands stay responsive. */
|
||||||
|
private const val MAX_WAIT_US = 20_000L
|
||||||
|
/** A frame this far behind the clock is dropped undrawn... */
|
||||||
|
private const val LATE_DROP_US = 50_000L
|
||||||
|
/** ...unless nothing has been drawn for this long, so a slow phone still shows progress. */
|
||||||
|
private const val MAX_DROP_GAP_NANOS = 250_000_000L
|
||||||
|
/**
|
||||||
|
* Decoded frames closer together than this are thinned: past ~30fps the glasses link
|
||||||
|
* could never show them, and each one still costs a scale and a composite.
|
||||||
|
*/
|
||||||
|
private const val MIN_FRAME_INTERVAL_US = 30_000L
|
||||||
|
/** Audio position is ignored this long after the clock starts (output startup latency). */
|
||||||
|
private const val AUDIO_SETTLE_NANOS = 500_000_000L
|
||||||
|
/** Clock drift from the audio position that makes the video clock jump to it. */
|
||||||
|
private const val AUDIO_RESYNC_US = 60_000L
|
||||||
|
/** How often the audio position is read (a binder call into the media server). */
|
||||||
|
private const val AUDIO_CHECK_INTERVAL_NANOS = 250_000_000L
|
||||||
|
/** A seek never lands closer than this to the end, so it always shows a frame. */
|
||||||
|
private const val SEEK_END_MARGIN_US = 500_000L
|
||||||
|
|
||||||
|
private val AUDIO_ATTRIBUTES: AudioAttributes = AudioAttributes.Builder()
|
||||||
|
.setUsage(AudioAttributes.USAGE_MEDIA)
|
||||||
|
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
|
||||||
|
.build()
|
||||||
|
|
||||||
|
private fun MediaFormat.intOr(key: String, fallback: Int): Int =
|
||||||
|
if (containsKey(key)) getInteger(key) else fallback
|
||||||
|
|
||||||
|
private fun MediaFormat.longOr(key: String, fallback: Long): Long =
|
||||||
|
if (containsKey(key)) getLong(key) else fallback
|
||||||
|
|
||||||
|
private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int {
|
||||||
|
for (i in 0 until extractor.trackCount) {
|
||||||
|
val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
|
||||||
|
if (mime.startsWith(mimePrefix)) return i
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describe a video file for the settings panel, as JSON: {width, height} in display
|
||||||
|
* orientation (rotation applied), rotation, durationMs, hasAudio, mime; or {error}.
|
||||||
|
*/
|
||||||
|
@JvmStatic
|
||||||
|
fun probe(path: String?): String {
|
||||||
|
val result = JSONObject()
|
||||||
|
val extractor = MediaExtractor()
|
||||||
|
try {
|
||||||
|
extractor.setDataSource(path ?: throw IOException("no path"))
|
||||||
|
val videoTrack = findTrack(extractor, "video/")
|
||||||
|
if (videoTrack < 0) {
|
||||||
|
result.put("error", "No video track")
|
||||||
|
return result.toString()
|
||||||
|
}
|
||||||
|
var durationUs = 0L
|
||||||
|
for (i in 0 until extractor.trackCount) {
|
||||||
|
durationUs = Math.max(durationUs, extractor.getTrackFormat(i).longOr(MediaFormat.KEY_DURATION, 0L))
|
||||||
|
}
|
||||||
|
val format = extractor.getTrackFormat(videoTrack)
|
||||||
|
val rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
|
||||||
|
val codedWidth = format.intOr(MediaFormat.KEY_WIDTH, 0)
|
||||||
|
val codedHeight = format.intOr(MediaFormat.KEY_HEIGHT, 0)
|
||||||
|
val swap = rotation == 90 || rotation == 270
|
||||||
|
result.put("width", if (swap) codedHeight else codedWidth)
|
||||||
|
result.put("height", if (swap) codedWidth else codedHeight)
|
||||||
|
result.put("rotation", rotation)
|
||||||
|
result.put("durationMs", durationUs / 1000)
|
||||||
|
result.put("hasAudio", findTrack(extractor, "audio/") >= 0)
|
||||||
|
result.put("mime", format.getString(MediaFormat.KEY_MIME) ?: "")
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.w(TAG, "probe failed for $path", e)
|
||||||
|
result.put("error", e.message ?: e.toString())
|
||||||
|
} finally {
|
||||||
|
extractor.release()
|
||||||
|
}
|
||||||
|
return result.toString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What one start() call asked for; fixed for that playback's lifetime. */
|
||||||
|
private class Params(
|
||||||
|
val path: String,
|
||||||
|
val surfaceId: String,
|
||||||
|
val x: Int,
|
||||||
|
val y: Int,
|
||||||
|
val width: Int,
|
||||||
|
val height: Int,
|
||||||
|
val monochrome: Boolean,
|
||||||
|
val gamma: Float,
|
||||||
|
val audio: Boolean,
|
||||||
|
)
|
||||||
|
|
||||||
|
private enum class Wait { DUE, INTERRUPTED, STOP }
|
||||||
|
|
||||||
|
private val appContext: Context = context.applicationContext
|
||||||
|
private val mainHandler = Handler(Looper.getMainLooper())
|
||||||
|
private val audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||||
|
@Volatile private var listener: Listener? = null
|
||||||
|
|
||||||
|
private val lock = ReentrantLock()
|
||||||
|
private val wake = lock.newCondition()
|
||||||
|
|
||||||
|
// Everything below is guarded by lock.
|
||||||
|
/** Bumped by every start/stop; a playback thread whose generation is stale exits. */
|
||||||
|
private var generation = 0
|
||||||
|
private var params: Params? = null
|
||||||
|
/** The user's intent: playing or paused. */
|
||||||
|
private var playing = false
|
||||||
|
private var ended = false
|
||||||
|
/** A seek (or the initial start) is decoding toward its target; the clock waits for its frame. */
|
||||||
|
private var seeking = false
|
||||||
|
private var pendingSeekUs = -1L
|
||||||
|
private var durationUs = 0L
|
||||||
|
private var clockRunning = false
|
||||||
|
/** Media time while the clock is stopped. */
|
||||||
|
private var heldUs = 0L
|
||||||
|
private var clockBaseUs = 0L
|
||||||
|
private var clockBaseNanos = 0L
|
||||||
|
private var clockStartedNanos = 0L
|
||||||
|
private var lastAudioCheckNanos = 0L
|
||||||
|
private var audio: MediaPlayer? = null
|
||||||
|
private var hasAudioFocus = false
|
||||||
|
private var focusRequest: AudioFocusRequest? = null
|
||||||
|
/** The most recently decoded frame (params.width x params.height), shown or previewed. */
|
||||||
|
private var lastFrame: ByteArray? = null
|
||||||
|
/** Checksum of the frame last submitted, so an unchanged frame is not recomposited; -1 = none. */
|
||||||
|
private var lastSubmittedChecksum = -1L
|
||||||
|
private var submitFailures = 0
|
||||||
|
|
||||||
|
private val focusListener = AudioManager.OnAudioFocusChangeListener { change ->
|
||||||
|
when (change) {
|
||||||
|
AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> {
|
||||||
|
if (change == AudioManager.AUDIOFOCUS_LOSS) lock.withLock { hasAudioFocus = false }
|
||||||
|
// Another app (a call, a music player) took the audio: pause like any player would.
|
||||||
|
if (isPlaying()) {
|
||||||
|
pause()
|
||||||
|
notifyState(generationNow(), "paused", getPositionMs())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun setListener(listener: Listener?) {
|
||||||
|
this.listener = listener
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start playing [path] from [startMs] into the (rectX, rectY, width, height) rect of
|
||||||
|
* compositor surface [surfaceId], replacing any current playback. Decoding and audio setup
|
||||||
|
* happen on the playback thread; failures arrive as [Listener.onError].
|
||||||
|
*/
|
||||||
|
fun start(
|
||||||
|
path: String,
|
||||||
|
surfaceId: String,
|
||||||
|
rectX: Int,
|
||||||
|
rectY: Int,
|
||||||
|
width: Int,
|
||||||
|
height: Int,
|
||||||
|
monochrome: Boolean,
|
||||||
|
gamma: Float,
|
||||||
|
withAudio: Boolean,
|
||||||
|
startMs: Long,
|
||||||
|
) {
|
||||||
|
require(width > 0 && height > 0) { "bad video rect ${width}x$height" }
|
||||||
|
val p = Params(path, surfaceId, rectX, rectY, width, height, monochrome, gamma, withAudio)
|
||||||
|
val gen = lock.withLock {
|
||||||
|
endPlaybackLocked()
|
||||||
|
params = p
|
||||||
|
playing = true
|
||||||
|
ended = false
|
||||||
|
seeking = true
|
||||||
|
pendingSeekUs = Math.max(0L, startMs) * 1000
|
||||||
|
heldUs = pendingSeekUs
|
||||||
|
// The old frame may not match the new settings (size, monochrome).
|
||||||
|
lastFrame = null
|
||||||
|
lastSubmittedChecksum = -1
|
||||||
|
submitFailures = 0
|
||||||
|
generation
|
||||||
|
}
|
||||||
|
val thread = Thread({ Playback(gen, p).run() }, "FaceclawVideo")
|
||||||
|
thread.priority = Thread.NORM_PRIORITY + 1
|
||||||
|
thread.start()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pause; once this returns no further frame is submitted until [resume]. */
|
||||||
|
fun pause() {
|
||||||
|
lock.withLock {
|
||||||
|
if (!playing) return
|
||||||
|
playing = false
|
||||||
|
holdClockLocked(clockUsLocked())
|
||||||
|
wake.signalAll()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun resume() {
|
||||||
|
lock.withLock {
|
||||||
|
if (playing || ended || params == null) return
|
||||||
|
playing = true
|
||||||
|
// A fresh frame must go out even if it matches the last one: the TS side painted over
|
||||||
|
// the surface while paused.
|
||||||
|
lastSubmittedChecksum = -1
|
||||||
|
if (!seeking) startClockLocked()
|
||||||
|
wake.signalAll()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Seek to [positionMs]; while paused, the new frame is decoded and announced as "paused". */
|
||||||
|
fun seekTo(positionMs: Long) {
|
||||||
|
lock.withLock {
|
||||||
|
if (params == null || ended) return
|
||||||
|
var targetUs = Math.max(0L, positionMs) * 1000
|
||||||
|
if (durationUs > 0) targetUs = Math.min(targetUs, Math.max(0L, durationUs - SEEK_END_MARGIN_US))
|
||||||
|
pendingSeekUs = targetUs
|
||||||
|
seeking = true
|
||||||
|
holdClockLocked(targetUs)
|
||||||
|
lastSubmittedChecksum = -1
|
||||||
|
wake.signalAll()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getPositionMs(): Long = lock.withLock { clockUsLocked() / 1000 }
|
||||||
|
|
||||||
|
fun getDurationMs(): Long = lock.withLock { durationUs / 1000 }
|
||||||
|
|
||||||
|
fun isPlaying(): Boolean = lock.withLock { playing && !ended && params != null }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Copy the most recent frame into [out], which must hold exactly the video rect's
|
||||||
|
* width*height bytes (row-major 8bpp gray). False when there is no frame of that size yet.
|
||||||
|
*/
|
||||||
|
fun copyLastFrame(out: ByteBuffer): Boolean {
|
||||||
|
lock.withLock {
|
||||||
|
val frame = lastFrame ?: return false
|
||||||
|
if (out.capacity() != frame.size) return false
|
||||||
|
out.clear()
|
||||||
|
out.put(frame)
|
||||||
|
out.clear()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** End playback and free the decoder and audio; the last frame stays readable. */
|
||||||
|
fun stop() {
|
||||||
|
lock.withLock { endPlaybackLocked() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** stop(), and drop the listener and frame. */
|
||||||
|
fun release() {
|
||||||
|
lock.withLock {
|
||||||
|
endPlaybackLocked()
|
||||||
|
lastFrame = null
|
||||||
|
}
|
||||||
|
listener = null
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun generationNow(): Int = lock.withLock { generation }
|
||||||
|
|
||||||
|
private fun endPlaybackLocked() {
|
||||||
|
generation++
|
||||||
|
params = null
|
||||||
|
playing = false
|
||||||
|
seeking = false
|
||||||
|
pendingSeekUs = -1
|
||||||
|
// Keep the position readable (an error's caller resumes from it).
|
||||||
|
heldUs = clockUsLocked()
|
||||||
|
clockRunning = false
|
||||||
|
releaseAudioLocked()
|
||||||
|
wake.signalAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- clock (all under lock) ----
|
||||||
|
|
||||||
|
private fun clockUsLocked(): Long =
|
||||||
|
if (!clockRunning) heldUs else clockBaseUs + (System.nanoTime() - clockBaseNanos) / 1000
|
||||||
|
|
||||||
|
private fun startClockLocked() {
|
||||||
|
if (clockRunning) return
|
||||||
|
clockRunning = true
|
||||||
|
clockBaseUs = heldUs
|
||||||
|
clockBaseNanos = System.nanoTime()
|
||||||
|
clockStartedNanos = clockBaseNanos
|
||||||
|
val player = audio ?: return
|
||||||
|
requestAudioFocusLocked()
|
||||||
|
try {
|
||||||
|
player.start()
|
||||||
|
} catch (e: IllegalStateException) {
|
||||||
|
Log.w(TAG, "audio start failed", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun holdClockLocked(atUs: Long) {
|
||||||
|
heldUs = atUs
|
||||||
|
if (!clockRunning) return
|
||||||
|
clockRunning = false
|
||||||
|
try {
|
||||||
|
audio?.let { if (it.isPlaying) it.pause() }
|
||||||
|
} catch (e: IllegalStateException) {
|
||||||
|
Log.w(TAG, "audio pause failed", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep the video clock on the audio position: output latency and seeks make the two drift,
|
||||||
|
* and a picture ahead of or behind its sound is the one sync error people notice.
|
||||||
|
*/
|
||||||
|
private fun followAudioLocked() {
|
||||||
|
val player = audio ?: return
|
||||||
|
val now = System.nanoTime()
|
||||||
|
if (!clockRunning || now - clockStartedNanos < AUDIO_SETTLE_NANOS) return
|
||||||
|
if (now - lastAudioCheckNanos < AUDIO_CHECK_INTERVAL_NANOS) return
|
||||||
|
lastAudioCheckNanos = now
|
||||||
|
val audioUs = try {
|
||||||
|
if (!player.isPlaying) return // finished (audio shorter than video) or not started
|
||||||
|
player.currentPosition * 1000L
|
||||||
|
} catch (e: IllegalStateException) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (Math.abs(audioUs - clockUsLocked()) > AUDIO_RESYNC_US) {
|
||||||
|
clockBaseUs = audioUs
|
||||||
|
clockBaseNanos = System.nanoTime()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun seekAudioLocked(targetUs: Long) {
|
||||||
|
val player = audio ?: return
|
||||||
|
try {
|
||||||
|
if (Build.VERSION.SDK_INT >= 26) {
|
||||||
|
player.seekTo(targetUs / 1000, MediaPlayer.SEEK_CLOSEST)
|
||||||
|
} else {
|
||||||
|
player.seekTo((targetUs / 1000).toInt())
|
||||||
|
}
|
||||||
|
} catch (e: IllegalStateException) {
|
||||||
|
Log.w(TAG, "audio seek failed", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun requestAudioFocusLocked() {
|
||||||
|
if (hasAudioFocus) return
|
||||||
|
val result = if (Build.VERSION.SDK_INT >= 26) {
|
||||||
|
val request = focusRequest ?: AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN)
|
||||||
|
.setAudioAttributes(AUDIO_ATTRIBUTES)
|
||||||
|
.setOnAudioFocusChangeListener(focusListener, mainHandler)
|
||||||
|
.build()
|
||||||
|
.also { focusRequest = it }
|
||||||
|
audioManager.requestAudioFocus(request)
|
||||||
|
} else {
|
||||||
|
@Suppress("DEPRECATION")
|
||||||
|
audioManager.requestAudioFocus(focusListener, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN)
|
||||||
|
}
|
||||||
|
// Play regardless: the sound was asked for, and a refusal is transient (a call ending).
|
||||||
|
hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun releaseAudioLocked() {
|
||||||
|
val player = audio ?: return
|
||||||
|
audio = null
|
||||||
|
try {
|
||||||
|
player.release()
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.w(TAG, "audio release failed", e)
|
||||||
|
}
|
||||||
|
if (hasAudioFocus) {
|
||||||
|
hasAudioFocus = false
|
||||||
|
if (Build.VERSION.SDK_INT >= 26) {
|
||||||
|
focusRequest?.let { audioManager.abandonAudioFocusRequest(it) }
|
||||||
|
} else {
|
||||||
|
@Suppress("DEPRECATION")
|
||||||
|
audioManager.abandonAudioFocus(focusListener)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- notifications ----
|
||||||
|
|
||||||
|
private fun notifyState(gen: Int, state: String, positionMs: Long) {
|
||||||
|
mainHandler.post {
|
||||||
|
if (lock.withLock { gen == generation }) listener?.onStateChanged(state, positionMs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun notifyError(gen: Int, message: String) {
|
||||||
|
mainHandler.post {
|
||||||
|
if (lock.withLock { gen == generation }) listener?.onError(message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- frame output (under lock) ----
|
||||||
|
|
||||||
|
private fun submitLocked(p: Params, pixels: ByteArray, checksum: Long) {
|
||||||
|
val buffer = ByteBuffer.wrap(pixels)
|
||||||
|
// Frame id 0: per-frame FrameTimings entries at video rate would crowd every real
|
||||||
|
// input/render frame out of the timing export.
|
||||||
|
val fingerprint = "video:$generation:" + java.lang.Long.toHexString(checksum)
|
||||||
|
try {
|
||||||
|
val communicator = FaceclawBleCommunicator.getActive()
|
||||||
|
if (communicator != null) {
|
||||||
|
communicator.submitSurfaceFrame(buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
|
||||||
|
} else {
|
||||||
|
FaceclawPreviewCompositor.getActive()?.submitSurfaceFrame(
|
||||||
|
buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
|
||||||
|
}
|
||||||
|
submitFailures = 0
|
||||||
|
} catch (e: RuntimeException) {
|
||||||
|
// The window's surface went away or shrank under us (closed, relaid out): drop the frame.
|
||||||
|
if (submitFailures++ == 0) Log.w(TAG, "video frame submit failed: ${e.message}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One start() call's decode thread. Owns the extractor and decoder; everything shared with
|
||||||
|
* the control methods goes through the player's lock.
|
||||||
|
*/
|
||||||
|
private inner class Playback(private val gen: Int, private val p: Params) {
|
||||||
|
private var extractor: MediaExtractor? = null
|
||||||
|
private var codec: MediaCodec? = null
|
||||||
|
private val info = MediaCodec.BufferInfo()
|
||||||
|
private var inputDone = false
|
||||||
|
private var outputDone = false
|
||||||
|
private var trackEndUs = 0L
|
||||||
|
/** Frames before this are decoded only on the way to a seek target. */
|
||||||
|
private var skipUntilUs = 0L
|
||||||
|
private var lastDrawnPtsUs = Long.MIN_VALUE / 2
|
||||||
|
private var lastDrawnNanos = 0L
|
||||||
|
private var rotation = 0
|
||||||
|
private var lut = ByteArray(256)
|
||||||
|
private var scaler: VideoFrameScaler? = null
|
||||||
|
private var luma = ByteArray(0)
|
||||||
|
/** The frame being built; swapped with lastFrame when shown. */
|
||||||
|
private var frame = ByteArray(p.width * p.height)
|
||||||
|
private val crc = CRC32()
|
||||||
|
|
||||||
|
fun run() {
|
||||||
|
try {
|
||||||
|
open()
|
||||||
|
loop()
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
Log.w(TAG, "playback failed for ${p.path}", t)
|
||||||
|
val current = lock.withLock {
|
||||||
|
val current = gen == generation
|
||||||
|
if (current) endPlaybackLocked()
|
||||||
|
current
|
||||||
|
}
|
||||||
|
// endPlaybackLocked bumped the generation; report under the new one.
|
||||||
|
if (current) notifyError(generationNow(), t.message ?: t.toString())
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
codec?.stop()
|
||||||
|
} catch (ignored: Throwable) {
|
||||||
|
}
|
||||||
|
codec?.release()
|
||||||
|
extractor?.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun current(): Boolean = lock.withLock { gen == generation }
|
||||||
|
|
||||||
|
private fun open() {
|
||||||
|
val ex = MediaExtractor()
|
||||||
|
extractor = ex
|
||||||
|
ex.setDataSource(p.path)
|
||||||
|
val track = findTrack(ex, "video/")
|
||||||
|
if (track < 0) throw IOException("No video track")
|
||||||
|
ex.selectTrack(track)
|
||||||
|
val format = ex.getTrackFormat(track)
|
||||||
|
rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
|
||||||
|
trackEndUs = format.longOr(MediaFormat.KEY_DURATION, 0L)
|
||||||
|
// Most video is limited ("TV") range; only an explicit full-range tag says otherwise.
|
||||||
|
val limited = format.intOr(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) !=
|
||||||
|
MediaFormat.COLOR_RANGE_FULL
|
||||||
|
lut = VideoFrameScaler.toneLut(limited, p.gamma, p.monochrome)
|
||||||
|
val mime = format.getString(MediaFormat.KEY_MIME) ?: throw IOException("Unknown video format")
|
||||||
|
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
|
||||||
|
val decoder = try {
|
||||||
|
MediaCodec.createDecoderByType(mime)
|
||||||
|
} catch (e: Exception) {
|
||||||
|
throw IOException("No decoder for $mime")
|
||||||
|
}
|
||||||
|
codec = decoder
|
||||||
|
decoder.configure(format, null, null, 0)
|
||||||
|
decoder.start()
|
||||||
|
val player = if (p.audio && findTrack(ex, "audio/") >= 0) openAudio() else null
|
||||||
|
lock.withLock {
|
||||||
|
if (gen != generation) {
|
||||||
|
player?.release()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
durationUs = trackEndUs
|
||||||
|
// Positioned by the start seek, which is the first command loop() handles.
|
||||||
|
audio = player
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A prepared MediaPlayer for the file's audio, or null (played silent) if it won't open. */
|
||||||
|
private fun openAudio(): MediaPlayer? {
|
||||||
|
val player = MediaPlayer()
|
||||||
|
return try {
|
||||||
|
player.setAudioAttributes(AUDIO_ATTRIBUTES)
|
||||||
|
player.setDataSource(p.path)
|
||||||
|
player.prepare()
|
||||||
|
player
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.w(TAG, "audio unavailable for ${p.path}; playing silent", e)
|
||||||
|
player.release()
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun loop() {
|
||||||
|
while (true) {
|
||||||
|
// Commands: stop, seek, or wait out a pause.
|
||||||
|
var seekUs = -1L
|
||||||
|
lock.withLock {
|
||||||
|
while (true) {
|
||||||
|
if (gen != generation) return
|
||||||
|
if (pendingSeekUs >= 0) {
|
||||||
|
seekUs = pendingSeekUs
|
||||||
|
pendingSeekUs = -1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (playing || seeking) break
|
||||||
|
wake.await()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (seekUs >= 0) {
|
||||||
|
seek(seekUs)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (outputDone) {
|
||||||
|
if (awaitEnd()) return
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
feedInput()
|
||||||
|
val index = codec!!.dequeueOutputBuffer(info, CODEC_TIMEOUT_US)
|
||||||
|
if (index < 0) continue // try again later / format or buffers changed
|
||||||
|
val endOfStream = (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0
|
||||||
|
if (info.size > 0) {
|
||||||
|
handleFrame(index, info.presentationTimeUs)
|
||||||
|
} else {
|
||||||
|
codec!!.releaseOutputBuffer(index, false)
|
||||||
|
}
|
||||||
|
if (endOfStream) outputDone = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun seek(targetUs: Long) {
|
||||||
|
lock.withLock {
|
||||||
|
if (gen != generation) return
|
||||||
|
seekAudioLocked(targetUs)
|
||||||
|
}
|
||||||
|
extractor!!.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
|
||||||
|
codec!!.flush()
|
||||||
|
inputDone = false
|
||||||
|
outputDone = false
|
||||||
|
skipUntilUs = targetUs
|
||||||
|
lastDrawnPtsUs = Long.MIN_VALUE / 2
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun feedInput() {
|
||||||
|
if (inputDone) return
|
||||||
|
val decoder = codec!!
|
||||||
|
val ex = extractor!!
|
||||||
|
while (true) {
|
||||||
|
val index = decoder.dequeueInputBuffer(0)
|
||||||
|
if (index < 0) return
|
||||||
|
val buffer = decoder.getInputBuffer(index) ?: return
|
||||||
|
val size = ex.readSampleData(buffer, 0)
|
||||||
|
if (size < 0) {
|
||||||
|
decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||||
|
inputDone = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
decoder.queueInputBuffer(index, 0, size, ex.sampleTime, 0)
|
||||||
|
ex.advance()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decide a decoded frame's fate (skip, preview, drop, draw); always releases it. */
|
||||||
|
private fun handleFrame(index: Int, ptsUs: Long) {
|
||||||
|
try {
|
||||||
|
if (ptsUs < skipUntilUs) return
|
||||||
|
val landingSeek = lock.withLock { seeking }
|
||||||
|
if (landingSeek) {
|
||||||
|
landSeek(index, ptsUs)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (ptsUs - lastDrawnPtsUs < MIN_FRAME_INTERVAL_US) return
|
||||||
|
if (waitUntilDue(ptsUs) != Wait.DUE) return
|
||||||
|
val lateUs = lock.withLock { clockUsLocked() } - ptsUs
|
||||||
|
if (lateUs > LATE_DROP_US && System.nanoTime() - lastDrawnNanos < MAX_DROP_GAP_NANOS) return
|
||||||
|
val checksum = convert(index)
|
||||||
|
present(ptsUs, checksum)
|
||||||
|
} finally {
|
||||||
|
codec!!.releaseOutputBuffer(index, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The first frame at or past a seek target (or the start position): drawn and the clock
|
||||||
|
* started when playing, kept as the paused preview otherwise.
|
||||||
|
*/
|
||||||
|
private fun landSeek(index: Int, ptsUs: Long) {
|
||||||
|
val checksum = convert(index)
|
||||||
|
var previewed = false
|
||||||
|
lock.withLock {
|
||||||
|
if (gen != generation || pendingSeekUs >= 0) return
|
||||||
|
seeking = false
|
||||||
|
heldUs = ptsUs
|
||||||
|
swapInFrameLocked()
|
||||||
|
if (playing) {
|
||||||
|
submitLocked(p, lastFrame!!, checksum)
|
||||||
|
lastSubmittedChecksum = checksum
|
||||||
|
startClockLocked()
|
||||||
|
} else {
|
||||||
|
previewed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lastDrawnPtsUs = ptsUs
|
||||||
|
lastDrawnNanos = System.nanoTime()
|
||||||
|
notifyState(gen, if (previewed) "paused" else "playing", ptsUs / 1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Show a due frame, unless a pause or seek got in first. */
|
||||||
|
private fun present(ptsUs: Long, checksum: Long) {
|
||||||
|
lock.withLock {
|
||||||
|
if (gen != generation || !playing || pendingSeekUs >= 0) return
|
||||||
|
swapInFrameLocked()
|
||||||
|
if (checksum != lastSubmittedChecksum) {
|
||||||
|
submitLocked(p, lastFrame!!, checksum)
|
||||||
|
lastSubmittedChecksum = checksum
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lastDrawnPtsUs = ptsUs
|
||||||
|
lastDrawnNanos = System.nanoTime()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun swapInFrameLocked() {
|
||||||
|
val shown = frame
|
||||||
|
val previous = lastFrame
|
||||||
|
frame = if (previous != null && previous.size == shown.size) previous else ByteArray(shown.size)
|
||||||
|
lastFrame = shown
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Block until the clock reaches [ptsUs]; paused time doesn't count. */
|
||||||
|
private fun waitUntilDue(ptsUs: Long): Wait {
|
||||||
|
lock.withLock {
|
||||||
|
while (true) {
|
||||||
|
if (gen != generation) return Wait.STOP
|
||||||
|
if (pendingSeekUs >= 0) return Wait.INTERRUPTED
|
||||||
|
if (!playing) {
|
||||||
|
wake.await()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
followAudioLocked()
|
||||||
|
val now = clockUsLocked()
|
||||||
|
if (now >= ptsUs) return Wait.DUE
|
||||||
|
wake.await(Math.min(ptsUs - now, MAX_WAIT_US), TimeUnit.MICROSECONDS)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The decoder is drained: hold the last frame until the clock reaches the end, then
|
||||||
|
* report "ended". False when a seek interrupted the wait (playback continues).
|
||||||
|
*/
|
||||||
|
private fun awaitEnd(): Boolean {
|
||||||
|
val endUs = Math.max(trackEndUs, lastDrawnPtsUs)
|
||||||
|
val landing = lock.withLock { seeking }
|
||||||
|
if (!landing && waitUntilDue(endUs) != Wait.DUE) return !current()
|
||||||
|
lock.withLock {
|
||||||
|
if (gen != generation) return true
|
||||||
|
if (pendingSeekUs >= 0) return false
|
||||||
|
playing = false
|
||||||
|
seeking = false
|
||||||
|
ended = true
|
||||||
|
holdClockLocked(endUs)
|
||||||
|
releaseAudioLocked()
|
||||||
|
}
|
||||||
|
notifyState(gen, "ended", endUs / 1000)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scale the output buffer's frame into [frame]; returns its checksum. */
|
||||||
|
private fun convert(index: Int): Long {
|
||||||
|
val decoder = codec!!
|
||||||
|
val image = try {
|
||||||
|
decoder.getOutputImage(index)
|
||||||
|
} catch (e: Exception) {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
if (image != null) {
|
||||||
|
try {
|
||||||
|
val plane = image.planes[0]
|
||||||
|
val crop = image.cropRect
|
||||||
|
val offset = copyLuma(plane.buffer) + crop.top * plane.rowStride + crop.left * plane.pixelStride
|
||||||
|
scaleInto(crop.width(), crop.height(), offset, plane.rowStride, plane.pixelStride)
|
||||||
|
} finally {
|
||||||
|
image.close()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No Image access: every YUV420 layout starts with the Y plane at the buffer start.
|
||||||
|
val buffer = decoder.getOutputBuffer(index) ?: throw IOException("Decoder returned no frame")
|
||||||
|
buffer.position(info.offset)
|
||||||
|
buffer.limit(info.offset + info.size)
|
||||||
|
val format = decoder.outputFormat
|
||||||
|
val width = format.getInteger(MediaFormat.KEY_WIDTH)
|
||||||
|
val stride = format.intOr(MediaFormat.KEY_STRIDE, width)
|
||||||
|
val left = format.intOr("crop-left", 0)
|
||||||
|
val top = format.intOr("crop-top", 0)
|
||||||
|
val right = format.intOr("crop-right", width - 1)
|
||||||
|
val bottom = format.intOr("crop-bottom", format.getInteger(MediaFormat.KEY_HEIGHT) - 1)
|
||||||
|
copyLuma(buffer)
|
||||||
|
scaleInto(right - left + 1, bottom - top + 1, top * stride + left, stride, 1)
|
||||||
|
}
|
||||||
|
crc.reset()
|
||||||
|
crc.update(frame, 0, frame.size)
|
||||||
|
return crc.value
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bulk-copy the plane (one memcpy; per-byte reads from a direct buffer are slow). Returns 0. */
|
||||||
|
private fun copyLuma(buffer: ByteBuffer): Int {
|
||||||
|
val bytes = buffer.remaining()
|
||||||
|
if (luma.size < bytes) luma = ByteArray(bytes)
|
||||||
|
buffer.get(luma, 0, bytes)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun scaleInto(width: Int, height: Int, offset: Int, rowStride: Int, pixelStride: Int) {
|
||||||
|
var s = scaler
|
||||||
|
if (s == null || s.sourceWidth != width || s.sourceHeight != height) {
|
||||||
|
s = VideoFrameScaler(width, height, rotation, p.width, p.height)
|
||||||
|
scaler = s
|
||||||
|
}
|
||||||
|
s.scale(luma, offset, rowStride, pixelStride, lut, frame)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
|||||||
import { isFontFile } from "../../native/font-files";
|
import { isFontFile } from "../../native/font-files";
|
||||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||||
import { isDecodableImageFile } from "../../native/image-files";
|
import { isDecodableImageFile } from "../../native/image-files";
|
||||||
|
import { isPlayableVideoFile } from "../../native/video-player";
|
||||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||||
import { FileBrowserLayer } from "./file-browser";
|
import { FileBrowserLayer } from "./file-browser";
|
||||||
@@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & {
|
|||||||
installEhpkApp: (path: string) => Promise<void> | void;
|
installEhpkApp: (path: string) => Promise<void> | void;
|
||||||
/** Open a font file's previewer as its own shell window. */
|
/** Open a font file's previewer as its own shell window. */
|
||||||
openFontWindow: (title: string, path: string) => void;
|
openFontWindow: (title: string, path: string) => void;
|
||||||
|
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||||
|
openVideoWindow: (title: string, path: string) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Files app's launcher-opened window: a file browser over Places
|
* The Files app's launcher-opened window: a file browser over Places
|
||||||
* (bookmarks and storage roots). Picking any file opens an info dialog with
|
* (bookmarks and storage roots). Picking any file opens an info dialog with
|
||||||
* metadata plus, for viewable types (text, images), the open actions.
|
* metadata plus, for viewable types (text, images, video), the open actions.
|
||||||
*/
|
*/
|
||||||
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
|
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
|
||||||
let created: InProcessWindow | null = null;
|
let created: InProcessWindow | null = null;
|
||||||
const browser = new FileBrowserLayer({
|
const browser = new FileBrowserLayer({
|
||||||
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)),
|
isSupportedFile: (name) =>
|
||||||
|
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||||
|
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||||
// The browser handles double-click itself (up a level), so it is not
|
// The browser handles double-click itself (up a level), so it is not
|
||||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||||
onLeave: () => shell.yieldFocusToSidebar(),
|
onLeave: () => shell.yieldFocusToSidebar(),
|
||||||
@@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
if (isPlayableVideoFile(entry.name)) {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
label: "Play video",
|
||||||
|
onSelect: (ctx) => {
|
||||||
|
ctx.stack.pop();
|
||||||
|
options.openVideoWindow(entry.name, entry.path);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
if (isFontFile(entry.name)) {
|
if (isFontFile(entry.name)) {
|
||||||
// Install copies into the app-internal fonts directory, making the face
|
// Install copies into the app-internal fonts directory, making the face
|
||||||
// available in the Settings font pickers. The row relabels itself with
|
// available in the Settings font pickers. The row relabels itself with
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
FILES_SURFACE_ID,
|
FILES_SURFACE_ID,
|
||||||
FILES_WINDOW_ID,
|
FILES_WINDOW_ID,
|
||||||
} from "./files-app";
|
} from "./files-app";
|
||||||
|
import { createVideoPlayerWindow } from "./video-window";
|
||||||
|
|
||||||
// Document windows are closeable and non-singleton; the serial keeps their
|
// Document windows are closeable and non-singleton; the serial keeps their
|
||||||
// windowIds unique.
|
// windowIds unique.
|
||||||
@@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function openVideoWindow(ctx: AppContext, title: string, path: string): void {
|
||||||
|
const windowId = `files:doc:${nextDocumentSerial++}`;
|
||||||
|
void ctx
|
||||||
|
.launchInProcessApp(windowId, `window:${windowId}`, (options) =>
|
||||||
|
createVideoPlayerWindow(windowId, title, path, options),
|
||||||
|
)
|
||||||
|
.catch((error) => {
|
||||||
|
ctx.appendLog(`video window failed: ${error}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const filesApp: AppDefinition = {
|
const filesApp: AppDefinition = {
|
||||||
appId: "files",
|
appId: "files",
|
||||||
title: "Files",
|
title: "Files",
|
||||||
@@ -78,6 +90,7 @@ const filesApp: AppDefinition = {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||||
|
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
// A document arriving via Android's Share intent opens as its own window.
|
// A document arriving via Android's Share intent opens as its own window.
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
/**
|
||||||
|
* Where a video goes in the player window, for each size choice. Pure
|
||||||
|
* geometry, so it is unit-tested without the NativeScript runtime.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type VideoSize = "small" | "medium" | "large" | "full";
|
||||||
|
|
||||||
|
export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"];
|
||||||
|
|
||||||
|
export type Size = { width: number; height: number };
|
||||||
|
export type Rect = Size & { x: number; y: number };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bounding boxes for the fixed sizes; a video is fitted inside, keeping its
|
||||||
|
* aspect. Large and full fit the whole viewport instead (full in a tall
|
||||||
|
* window). Bigger pictures cost the glasses link more per frame, so smaller
|
||||||
|
* sizes play at a higher frame rate.
|
||||||
|
*/
|
||||||
|
const FIXED_BOXES: Partial<Record<VideoSize, Size>> = {
|
||||||
|
small: { width: 160, height: 120 },
|
||||||
|
medium: { width: 240, height: 180 },
|
||||||
|
};
|
||||||
|
|
||||||
|
export function videoSizeLabel(size: VideoSize): string {
|
||||||
|
switch (size) {
|
||||||
|
case "small":
|
||||||
|
return "Small";
|
||||||
|
case "medium":
|
||||||
|
return "Medium";
|
||||||
|
case "large":
|
||||||
|
return "Large";
|
||||||
|
case "full":
|
||||||
|
return "Full screen";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Full screen needs the tall window band; the rest play in the standard one. */
|
||||||
|
export function videoHeightMode(size: VideoSize): "min" | "max" {
|
||||||
|
return size === "full" ? "max" : "min";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The largest size with the video's aspect ratio that fits in box, with even
|
||||||
|
* dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel
|
||||||
|
* columns) of at least 2. Upscales small videos too: the size is the
|
||||||
|
* viewer's choice.
|
||||||
|
*/
|
||||||
|
export function fitVideo(video: Size, box: Size): Size {
|
||||||
|
if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) };
|
||||||
|
const scale = Math.min(box.width / video.width, box.height / video.height);
|
||||||
|
return {
|
||||||
|
width: evenFloor(Math.min(box.width, Math.round(video.width * scale))),
|
||||||
|
height: evenFloor(Math.min(box.height, Math.round(video.height * scale))),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The video's rect within a viewport, centered. x snaps to a multiple of 4
|
||||||
|
* and y to a multiple of 2 so frame deltas don't straddle an extra encoder
|
||||||
|
* block at each edge.
|
||||||
|
*/
|
||||||
|
export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect {
|
||||||
|
const fixed = FIXED_BOXES[size];
|
||||||
|
const box = fixed
|
||||||
|
? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) }
|
||||||
|
: viewport;
|
||||||
|
const fitted = fitVideo(video, box);
|
||||||
|
return {
|
||||||
|
x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4),
|
||||||
|
y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2),
|
||||||
|
...fitted,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "3:52", or "1:02:03" past an hour. */
|
||||||
|
export function formatMediaTime(ms: number): string {
|
||||||
|
const total = Math.max(0, Math.floor(ms / 1000));
|
||||||
|
const hours = Math.floor(total / 3600);
|
||||||
|
const minutes = Math.floor((total % 3600) / 60);
|
||||||
|
const seconds = String(total % 60).padStart(2, "0");
|
||||||
|
return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function evenFloor(value: number): number {
|
||||||
|
return Math.max(2, Math.floor(value / 2) * 2);
|
||||||
|
}
|
||||||
@@ -0,0 +1,443 @@
|
|||||||
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
|
import { GrayImage } from "../../graphics/image";
|
||||||
|
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||||
|
import { PHOTO_GAMMA } from "../../native/image-files";
|
||||||
|
import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player";
|
||||||
|
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
|
||||||
|
import { type InputEvent } from "../../ui/gestures";
|
||||||
|
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||||
|
import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu";
|
||||||
|
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||||
|
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
|
||||||
|
import {
|
||||||
|
createInProcessWindow,
|
||||||
|
YieldAtRootLayer,
|
||||||
|
type InProcessAppOptions,
|
||||||
|
type InProcessWindow,
|
||||||
|
} from "../../ui/shell/in-process-window";
|
||||||
|
import { appViewportSize } from "../../ui/shell/geometry";
|
||||||
|
import {
|
||||||
|
formatMediaTime,
|
||||||
|
VIDEO_SIZES,
|
||||||
|
videoHeightMode,
|
||||||
|
videoRect,
|
||||||
|
videoSizeLabel,
|
||||||
|
type Rect,
|
||||||
|
type VideoSize,
|
||||||
|
} from "./video-layout";
|
||||||
|
|
||||||
|
export const videoSizeSetting = new ConfigSettingEnum<VideoSize>({
|
||||||
|
id: "video-size",
|
||||||
|
label: "Size",
|
||||||
|
storageKey: "video.size",
|
||||||
|
defaultValue: "large",
|
||||||
|
values: VIDEO_SIZES,
|
||||||
|
formatValue: videoSizeLabel,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const videoAudioSetting = new ConfigSettingBoolean({
|
||||||
|
id: "video-audio",
|
||||||
|
label: "Audio",
|
||||||
|
storageKey: "video.audio",
|
||||||
|
defaultValue: true,
|
||||||
|
description: "Play the video's sound on the phone's current audio output.",
|
||||||
|
});
|
||||||
|
|
||||||
|
export const videoMonochromeSetting = new ConfigSettingBoolean({
|
||||||
|
id: "video-monochrome",
|
||||||
|
label: "Monochrome",
|
||||||
|
storageKey: "video.monochrome",
|
||||||
|
defaultValue: false,
|
||||||
|
description: "Show only black and white: far less to send per frame, so a higher frame rate.",
|
||||||
|
});
|
||||||
|
|
||||||
|
/** How far one ring scroll seeks. */
|
||||||
|
const SEEK_STEP_MS = 10_000;
|
||||||
|
const MARGIN_X = 18;
|
||||||
|
const TITLE_Y = 10;
|
||||||
|
const PANEL_WIDTH = 340;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One opened video: its probe result, the native player, and the window
|
||||||
|
* glue. The window's base layer is the settings panel; playing pushes the
|
||||||
|
* playback layer, and backing out of it (or the video ending) returns to the
|
||||||
|
* panel with the position kept for Resume.
|
||||||
|
*/
|
||||||
|
class VideoSession {
|
||||||
|
readonly probe: VideoInfo | { error: string };
|
||||||
|
private player: VideoPlayer | null = null;
|
||||||
|
private window: InProcessWindow | null = null;
|
||||||
|
private playback: VideoPlaybackLayer | null = null;
|
||||||
|
/** Where Resume starts; 0 = from the beginning. */
|
||||||
|
resumeMs = 0;
|
||||||
|
/** The last playback error, shown on the panel until the next Play. */
|
||||||
|
error: string | null = null;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
readonly path: string,
|
||||||
|
readonly title: string,
|
||||||
|
readonly surfaceId: string,
|
||||||
|
) {
|
||||||
|
this.probe = probeVideo(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
get info(): VideoInfo | null {
|
||||||
|
return "error" in this.probe ? null : this.probe;
|
||||||
|
}
|
||||||
|
|
||||||
|
attach(window: InProcessWindow): void {
|
||||||
|
this.window = window;
|
||||||
|
}
|
||||||
|
|
||||||
|
isPlaying(): boolean {
|
||||||
|
return this.player?.isPlaying() ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
positionMs(): number {
|
||||||
|
return this.player?.positionMs() ?? this.resumeMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Current rect for a size choice, in the viewport of the band that size plays in. */
|
||||||
|
rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect {
|
||||||
|
const info = this.info;
|
||||||
|
return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport);
|
||||||
|
}
|
||||||
|
|
||||||
|
play(ctx: LayerContext, fromStart: boolean): void {
|
||||||
|
const window = this.window;
|
||||||
|
const info = this.info;
|
||||||
|
if (!window || !info) return;
|
||||||
|
this.error = null;
|
||||||
|
const size = videoSizeSetting.get();
|
||||||
|
window.setHeightMode(videoHeightMode(size));
|
||||||
|
const viewport = ctx.stack.getBaseSize();
|
||||||
|
const rect = this.rectFor(size, viewport);
|
||||||
|
const player = this.ensurePlayer();
|
||||||
|
this.playback = new VideoPlaybackLayer(this, rect, viewport);
|
||||||
|
ctx.stack.push(this.playback);
|
||||||
|
player.start({
|
||||||
|
path: this.path,
|
||||||
|
surfaceId: this.surfaceId,
|
||||||
|
rect,
|
||||||
|
monochrome: videoMonochromeSetting.get(),
|
||||||
|
gamma: PHOTO_GAMMA,
|
||||||
|
audio: info.hasAudio && videoAudioSetting.get(),
|
||||||
|
startMs: fromStart ? 0 : this.resumeMs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
togglePause(): void {
|
||||||
|
const player = this.player;
|
||||||
|
if (!player) return;
|
||||||
|
if (player.isPlaying()) player.pause();
|
||||||
|
else player.resume();
|
||||||
|
}
|
||||||
|
|
||||||
|
seekBy(deltaMs: number): void {
|
||||||
|
const player = this.player;
|
||||||
|
if (!player) return;
|
||||||
|
const duration = this.info?.durationMs ?? 0;
|
||||||
|
const target = Math.max(0, player.positionMs() + deltaMs);
|
||||||
|
player.seekTo(duration > 0 ? Math.min(target, duration) : target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The window went to the background or the screen went off: stop, keep the place. */
|
||||||
|
pauseForHost(): void {
|
||||||
|
if (!this.player?.isPlaying()) return;
|
||||||
|
this.player.pause();
|
||||||
|
// Paint the paused overlay now, so the retained surface shows it on return.
|
||||||
|
this.window?.requestRender();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Leave the playback layer for the panel, remembering where to resume. */
|
||||||
|
returnToPanel(resumeMs: number): void {
|
||||||
|
const player = this.player;
|
||||||
|
if (player) {
|
||||||
|
player.pause();
|
||||||
|
player.stop();
|
||||||
|
}
|
||||||
|
this.resumeMs = resumeMs;
|
||||||
|
const window = this.window;
|
||||||
|
const playback = this.playback;
|
||||||
|
this.playback = null;
|
||||||
|
if (window && playback) window.stack.popThrough(playback);
|
||||||
|
window?.setHeightMode("min");
|
||||||
|
window?.requestRender();
|
||||||
|
}
|
||||||
|
|
||||||
|
lastFrame(rect: Rect): GrayImage | null {
|
||||||
|
return this.player?.lastFrame(rect.width, rect.height) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
release(): void {
|
||||||
|
this.player?.release();
|
||||||
|
this.player = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensurePlayer(): VideoPlayer {
|
||||||
|
if (!this.player) {
|
||||||
|
this.player = new VideoPlayer({
|
||||||
|
onState: (state, positionMs) => this.onPlayerState(state, positionMs),
|
||||||
|
onError: (message) => {
|
||||||
|
this.error = message;
|
||||||
|
this.returnToPanel(this.positionMs());
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return this.player;
|
||||||
|
}
|
||||||
|
|
||||||
|
private onPlayerState(state: VideoPlaybackState, _positionMs: number): void {
|
||||||
|
if (state === "ended") {
|
||||||
|
this.returnToPanel(0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A paused seek decoded its new frame: repaint it under the overlay.
|
||||||
|
if (state === "paused") this.window?.requestRender();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The pre-play settings panel (the window's root): picture size, audio and
|
||||||
|
* monochrome, all remembered for the next video, then Play (or Resume plus
|
||||||
|
* Play from start once a position is kept).
|
||||||
|
*/
|
||||||
|
class VideoSettingsLayer implements Layer {
|
||||||
|
private readonly menu = new Menu<PanelRow>({
|
||||||
|
wrap: false,
|
||||||
|
rowGap: 1,
|
||||||
|
highlight: { radius: 6 },
|
||||||
|
getHeight: () => listRowHeight(getDefaultSmallFont()),
|
||||||
|
draw: (args) => this.drawRow(args),
|
||||||
|
});
|
||||||
|
constructor(private readonly session: VideoSession) {}
|
||||||
|
|
||||||
|
private rows(): PanelRow[] {
|
||||||
|
if (!this.session.info) return [];
|
||||||
|
const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }];
|
||||||
|
rows.push({ kind: "play", fromStart: false });
|
||||||
|
if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true });
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
paint(ctx: LayerContext): GrayImage {
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
|
const image = new GrayImage(width, height, 0);
|
||||||
|
image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220);
|
||||||
|
const step = lineStep(font);
|
||||||
|
let y = TITLE_Y + step;
|
||||||
|
|
||||||
|
const info = this.session.info;
|
||||||
|
const errors: string[] = [];
|
||||||
|
if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`);
|
||||||
|
if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`);
|
||||||
|
if (info) {
|
||||||
|
const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"];
|
||||||
|
image.drawText(font, MARGIN_X, y, facts.join(" · "), 130);
|
||||||
|
y += step;
|
||||||
|
}
|
||||||
|
for (const error of errors) {
|
||||||
|
for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) {
|
||||||
|
image.drawText(font, MARGIN_X, y, line, 200);
|
||||||
|
y += step;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
y += 6;
|
||||||
|
|
||||||
|
const rows = this.rows();
|
||||||
|
const selected = this.menu.selectedItem;
|
||||||
|
const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1;
|
||||||
|
// Land on Resume/Play by default: the settings usually stay as they were.
|
||||||
|
const playIndex = rows.findIndex((row) => row.kind === "play");
|
||||||
|
this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined);
|
||||||
|
const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6));
|
||||||
|
this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) },
|
||||||
|
ctx.stack.isFocused());
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||||
|
switch (event.type) {
|
||||||
|
case "scroll-up":
|
||||||
|
case "scroll-down":
|
||||||
|
await this.menu.handleInput(event);
|
||||||
|
return;
|
||||||
|
case "click": {
|
||||||
|
const row = this.menu.selectedItem;
|
||||||
|
if (row) this.activate(row, ctx);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private activate(row: PanelRow, ctx: LayerContext): void {
|
||||||
|
switch (row.kind) {
|
||||||
|
case "size":
|
||||||
|
videoSizeSetting.set(videoSizeSetting.next());
|
||||||
|
return;
|
||||||
|
case "audio":
|
||||||
|
if (this.session.info?.hasAudio) videoAudioSetting.toggle();
|
||||||
|
return;
|
||||||
|
case "monochrome":
|
||||||
|
videoMonochromeSetting.toggle();
|
||||||
|
return;
|
||||||
|
case "play":
|
||||||
|
this.session.play(ctx, row.fromStart);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs<PanelRow>): void {
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
const textX = x + 10;
|
||||||
|
const textWidth = width - 20;
|
||||||
|
switch (item.kind) {
|
||||||
|
case "size": {
|
||||||
|
const size = videoSizeSetting.get();
|
||||||
|
// The picture size this choice gets in its own band (full screen plays in the tall window).
|
||||||
|
const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size)));
|
||||||
|
drawRightValueMenuItem(image, font, textX, y, textWidth, "Size",
|
||||||
|
`${videoSizeLabel(size)} ${rect.width}x${rect.height}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case "audio": {
|
||||||
|
const hasAudio = this.session.info?.hasAudio ?? false;
|
||||||
|
drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)",
|
||||||
|
hasAudio && videoAudioSetting.get(), selected);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case "monochrome":
|
||||||
|
drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected);
|
||||||
|
return;
|
||||||
|
case "play": {
|
||||||
|
const resumeMs = this.session.resumeMs;
|
||||||
|
const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play";
|
||||||
|
image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameRow(a: PanelRow, b: PanelRow): boolean {
|
||||||
|
return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The playing video. The native player draws frames into the video rect on
|
||||||
|
* its own; this layer paints only when the window renders (input, pause,
|
||||||
|
* focus): black around the latest frame, plus a status box while paused.
|
||||||
|
* Click pauses/resumes, scroll seeks 10s, double-click returns to the panel.
|
||||||
|
*/
|
||||||
|
class VideoPlaybackLayer implements Layer {
|
||||||
|
private leaving = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly session: VideoSession,
|
||||||
|
private readonly rect: Rect,
|
||||||
|
/** The viewport the rect was laid out in. */
|
||||||
|
private readonly viewport: { width: number; height: number },
|
||||||
|
) {}
|
||||||
|
|
||||||
|
paint(ctx: LayerContext): GrayImage {
|
||||||
|
const { width, height } = ctx.stack.getBaseSize();
|
||||||
|
if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) {
|
||||||
|
// The window was relaid out (display mode change) and the native player's rect no
|
||||||
|
// longer fits the surface: back out to the panel, which lays out afresh on Play.
|
||||||
|
this.leaving = true;
|
||||||
|
setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0);
|
||||||
|
}
|
||||||
|
const image = new GrayImage(width, height, 0);
|
||||||
|
const frame = this.session.lastFrame(this.rect);
|
||||||
|
if (frame) image.bitBlt(frame, this.rect.x, this.rect.y);
|
||||||
|
if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleInput(event: InputEvent): Promise<void> {
|
||||||
|
switch (event.type) {
|
||||||
|
case "click":
|
||||||
|
this.session.togglePause();
|
||||||
|
return;
|
||||||
|
case "scroll-up":
|
||||||
|
this.session.seekBy(-SEEK_STEP_MS);
|
||||||
|
return;
|
||||||
|
case "scroll-down":
|
||||||
|
this.session.seekBy(SEEK_STEP_MS);
|
||||||
|
return;
|
||||||
|
case "double-click":
|
||||||
|
this.session.returnToPanel(this.session.positionMs());
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "Paused 1:23 / 3:52" over a progress bar, below the picture when there is
|
||||||
|
* room (small sizes), else over its bottom edge.
|
||||||
|
*/
|
||||||
|
private drawPausedBox(image: GrayImage, loading: boolean): void {
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
const duration = this.session.info?.durationMs ?? 0;
|
||||||
|
const position = this.session.positionMs();
|
||||||
|
const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`;
|
||||||
|
const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24);
|
||||||
|
const boxHeight = font.lineHeight + 14;
|
||||||
|
const rect = this.rect;
|
||||||
|
const x = Math.round(rect.x + (rect.width - boxWidth) / 2);
|
||||||
|
const below = rect.y + rect.height + 4;
|
||||||
|
const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6);
|
||||||
|
image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6);
|
||||||
|
image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6);
|
||||||
|
image.drawText(font, x + 12, y + 5, text, 220);
|
||||||
|
if (!loading && duration > 0) {
|
||||||
|
const barWidth = boxWidth - 24;
|
||||||
|
image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40);
|
||||||
|
image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A video file's player as its own window, opened from the Files app. */
|
||||||
|
export function createVideoPlayerWindow(
|
||||||
|
windowId: string,
|
||||||
|
title: string,
|
||||||
|
path: string,
|
||||||
|
options: InProcessAppOptions,
|
||||||
|
): InProcessWindow {
|
||||||
|
const session = new VideoSession(path, title, `window:${windowId}`);
|
||||||
|
const window = createInProcessWindow({
|
||||||
|
appId: "files",
|
||||||
|
windowId,
|
||||||
|
title,
|
||||||
|
iconLetter: "V",
|
||||||
|
icon: "film",
|
||||||
|
closeable: true,
|
||||||
|
actions: options.actions,
|
||||||
|
baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)),
|
||||||
|
// A playing video is the thing being watched: no idle screen-off.
|
||||||
|
keepsScreenOn: () => session.isPlaying(),
|
||||||
|
onForegroundChanged: (foreground) => {
|
||||||
|
if (!foreground) session.pauseForHost();
|
||||||
|
},
|
||||||
|
setScreenOn: (on) => {
|
||||||
|
if (!on) session.pauseForHost();
|
||||||
|
},
|
||||||
|
submitFrame: options.submitFrame,
|
||||||
|
setSurfaceVisible: options.setSurfaceVisible,
|
||||||
|
removeSurface: options.removeSurface,
|
||||||
|
reconfigureSurface: options.reconfigureSurface,
|
||||||
|
onClosed: () => {
|
||||||
|
session.release();
|
||||||
|
options.onClosed();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
session.attach(window);
|
||||||
|
return window;
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
/**
|
||||||
|
* Bridge to the Android FaceclawVideoPlayer: decodes a video file and
|
||||||
|
* streams its frames, scaled to gray, straight into a rect of a window's
|
||||||
|
* compositor surface, with the audio on the phone's default media route.
|
||||||
|
* Android only; on iOS probeVideo reports an error and nothing plays.
|
||||||
|
*/
|
||||||
|
import { Utils } from "@nativescript/core";
|
||||||
|
import { GrayImage } from "../graphics/image";
|
||||||
|
|
||||||
|
declare const com: any;
|
||||||
|
declare const java: any;
|
||||||
|
declare const global: any;
|
||||||
|
|
||||||
|
/** Containers MediaExtractor opens (AVI is not one of them). */
|
||||||
|
const PLAYABLE_VIDEO = /\.(mp4|m4v|mkv|webm|3gp|mov)$/i;
|
||||||
|
|
||||||
|
export function isPlayableVideoFile(name: string): boolean {
|
||||||
|
return PLAYABLE_VIDEO.test(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VideoInfo = {
|
||||||
|
/** Display size, rotation already applied. */
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
durationMs: number;
|
||||||
|
hasAudio: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Read a video's size, duration and tracks; an error string when it can't be played. */
|
||||||
|
export function probeVideo(path: string): VideoInfo | { error: string } {
|
||||||
|
if (!global.isAndroid) return { error: "Video playback is not supported on this phone" };
|
||||||
|
try {
|
||||||
|
const raw = JSON.parse(String(com.faceclaw.app.FaceclawVideoPlayer.probe(path)));
|
||||||
|
if (raw.error) return { error: String(raw.error) };
|
||||||
|
return {
|
||||||
|
width: Number(raw.width) || 0,
|
||||||
|
height: Number(raw.height) || 0,
|
||||||
|
durationMs: Number(raw.durationMs) || 0,
|
||||||
|
hasAudio: Boolean(raw.hasAudio),
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return { error: String(error) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VideoPlaybackState = "playing" | "paused" | "ended";
|
||||||
|
|
||||||
|
export type VideoPlayerCallbacks = {
|
||||||
|
/** State change, or a paused seek that landed on a new frame (state "paused"). */
|
||||||
|
onState: (state: VideoPlaybackState, positionMs: number) => void;
|
||||||
|
onError: (message: string) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type VideoStartOptions = {
|
||||||
|
path: string;
|
||||||
|
surfaceId: string;
|
||||||
|
/** The video's rect in surface coordinates. */
|
||||||
|
rect: { x: number; y: number; width: number; height: number };
|
||||||
|
monochrome: boolean;
|
||||||
|
/** Tone curve for gray output (ignored for monochrome); see PHOTO_GAMMA. */
|
||||||
|
gamma: number;
|
||||||
|
audio: boolean;
|
||||||
|
startMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** One player; frames go to whatever surface the latest start() named. Main thread only. */
|
||||||
|
export class VideoPlayer {
|
||||||
|
private readonly player: any;
|
||||||
|
// The Java listener proxy must stay referenced or it gets GC'd.
|
||||||
|
private readonly listenerProxy: any;
|
||||||
|
// Reused direct buffer (and its JS view) for copyLastFrame; see java-direct-buffer.ts
|
||||||
|
// for why frames cross as a Java-allocated buffer.
|
||||||
|
private frameBuffer: any = null;
|
||||||
|
private frameView: Uint8Array | null = null;
|
||||||
|
|
||||||
|
constructor(callbacks: VideoPlayerCallbacks) {
|
||||||
|
this.player = new com.faceclaw.app.FaceclawVideoPlayer(Utils.android.getApplicationContext());
|
||||||
|
this.listenerProxy = new com.faceclaw.app.FaceclawVideoPlayer.Listener({
|
||||||
|
onStateChanged: (state: string, positionMs: number) => {
|
||||||
|
callbacks.onState(String(state) as VideoPlaybackState, Number(positionMs));
|
||||||
|
},
|
||||||
|
onError: (message: string) => callbacks.onError(String(message ?? "Playback failed")),
|
||||||
|
});
|
||||||
|
this.player.setListener(this.listenerProxy);
|
||||||
|
}
|
||||||
|
|
||||||
|
start(options: VideoStartOptions): void {
|
||||||
|
const { rect } = options;
|
||||||
|
this.player.start(
|
||||||
|
options.path,
|
||||||
|
options.surfaceId,
|
||||||
|
Math.round(rect.x),
|
||||||
|
Math.round(rect.y),
|
||||||
|
Math.round(rect.width),
|
||||||
|
Math.round(rect.height),
|
||||||
|
options.monochrome,
|
||||||
|
options.gamma,
|
||||||
|
options.audio,
|
||||||
|
Math.max(0, Math.round(options.startMs)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Once this returns, no more frames reach the surface until resume(). */
|
||||||
|
pause(): void {
|
||||||
|
this.player.pause();
|
||||||
|
}
|
||||||
|
|
||||||
|
resume(): void {
|
||||||
|
this.player.resume();
|
||||||
|
}
|
||||||
|
|
||||||
|
seekTo(positionMs: number): void {
|
||||||
|
this.player.seekTo(Math.max(0, Math.round(positionMs)));
|
||||||
|
}
|
||||||
|
|
||||||
|
positionMs(): number {
|
||||||
|
return Number(this.player.getPositionMs());
|
||||||
|
}
|
||||||
|
|
||||||
|
isPlaying(): boolean {
|
||||||
|
return Boolean(this.player.isPlaying());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The latest decoded frame at the given size, or null before one exists. */
|
||||||
|
lastFrame(width: number, height: number): GrayImage | null {
|
||||||
|
const bytes = width * height;
|
||||||
|
if (bytes <= 0) return null;
|
||||||
|
if (!this.frameView || this.frameView.length !== bytes) {
|
||||||
|
this.frameBuffer = java.nio.ByteBuffer.allocateDirect(bytes);
|
||||||
|
this.frameView = new Uint8Array((ArrayBuffer as any).from(this.frameBuffer));
|
||||||
|
}
|
||||||
|
if (!this.player.copyLastFrame(this.frameBuffer)) return null;
|
||||||
|
const image = new GrayImage(width, height, 0);
|
||||||
|
image.pixels.set(this.frameView);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** End playback and free the decoder and audio (the last frame stays readable). */
|
||||||
|
stop(): void {
|
||||||
|
this.player.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
release(): void {
|
||||||
|
this.player.release();
|
||||||
|
this.frameBuffer = null;
|
||||||
|
this.frameView = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -44,6 +44,10 @@ export type InProcessWindowOptions = {
|
|||||||
onSystemMenuOpened?: () => void;
|
onSystemMenuOpened?: () => void;
|
||||||
onAppMenuOpened?: () => void;
|
onAppMenuOpened?: () => void;
|
||||||
setScreenOn?: ShellWindow["setScreenOn"];
|
setScreenOn?: ShellWindow["setScreenOn"];
|
||||||
|
/** See ShellWindow.keepsScreenOn. */
|
||||||
|
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||||
|
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||||
|
onForegroundChanged?: (foreground: boolean) => void;
|
||||||
/** Shared actions; requestRender is rebound to this window's render. */
|
/** Shared actions; requestRender is rebound to this window's render. */
|
||||||
actions: LayerActions;
|
actions: LayerActions;
|
||||||
/**
|
/**
|
||||||
@@ -162,6 +166,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
acceptsDirectional: true,
|
acceptsDirectional: true,
|
||||||
holdToTalk: options.holdToTalk,
|
holdToTalk: options.holdToTalk,
|
||||||
isVoiceCapturing: options.isVoiceCapturing,
|
isVoiceCapturing: options.isVoiceCapturing,
|
||||||
|
keepsScreenOn: options.keepsScreenOn,
|
||||||
setScreenOn: options.setScreenOn,
|
setScreenOn: options.setScreenOn,
|
||||||
heightMode,
|
heightMode,
|
||||||
hasAppMenu: () => appMenuItems().length > 0,
|
hasAppMenu: () => appMenuItems().length > 0,
|
||||||
@@ -218,6 +223,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
onFocus: options.onFocus,
|
onFocus: options.onFocus,
|
||||||
setForeground: (foreground) => {
|
setForeground: (foreground) => {
|
||||||
options.setSurfaceVisible(foreground);
|
options.setSurfaceVisible(foreground);
|
||||||
|
options.onForegroundChanged?.(foreground);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
const setHeightMode = (mode: WindowHeightMode) => {
|
const setHeightMode = (mode: WindowHeightMode) => {
|
||||||
|
|||||||
+10
-1
@@ -101,6 +101,12 @@ export type ShellWindow = {
|
|||||||
holdToTalk?: boolean;
|
holdToTalk?: boolean;
|
||||||
/** A window owns microphone capture outside the shell voice dialog. */
|
/** A window owns microphone capture outside the shell voice dialog. */
|
||||||
isVoiceCapturing?: () => boolean;
|
isVoiceCapturing?: () => boolean;
|
||||||
|
/**
|
||||||
|
* True while the window shows something the wearer watches without
|
||||||
|
* touching anything (a playing video): the idle screen timeout waits, as
|
||||||
|
* it does for voice capture, while this window is in the foreground.
|
||||||
|
*/
|
||||||
|
keepsScreenOn?: () => boolean;
|
||||||
/**
|
/**
|
||||||
* True when the window gives swipe-left / swipe-right (watch directional
|
* True when the window gives swipe-left / swipe-right (watch directional
|
||||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||||
@@ -647,7 +653,10 @@ class Shell {
|
|||||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||||
// tool loop can run for a while with no input); once the turn ends and
|
// tool loop can run for a while with no input); once the turn ends and
|
||||||
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
|
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
|
||||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
// A foreground window that is playing video holds it off the same way.
|
||||||
|
const foreground = this.foregroundWindow();
|
||||||
|
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
|
||||||
|
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
|
||||||
this.lastInputAtMs = nowMs;
|
this.lastInputAtMs = nowMs;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
+133
@@ -0,0 +1,133 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
import kotlin.jvm.JvmStatic
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scales one decoded video frame's luma plane into an 8bpp gray rect for a compositor surface:
|
||||||
|
* a box filter (averaging up to [MAX_TAPS] x [MAX_TAPS] evenly spaced samples per output pixel,
|
||||||
|
* nearest-neighbor when upscaling), then the container's display rotation, then a tone lookup.
|
||||||
|
*
|
||||||
|
* Only luma is read: the glasses show gray, and Y is exactly the gray a decoder hands back
|
||||||
|
* without any color conversion. Platform code owns decoding and copies the plane into a
|
||||||
|
* ByteArray; this class owns the pixel math. One instance serves one (source size, rotation,
|
||||||
|
* output size) combination and reuses its scratch buffers across frames; not thread-safe.
|
||||||
|
*/
|
||||||
|
class VideoFrameScaler(
|
||||||
|
/** Visible luma size in coded (unrotated) orientation. */
|
||||||
|
val sourceWidth: Int,
|
||||||
|
val sourceHeight: Int,
|
||||||
|
/** Clockwise display rotation in degrees: 0, 90, 180 or 270. */
|
||||||
|
val rotation: Int,
|
||||||
|
/** Output size in display orientation. */
|
||||||
|
val outputWidth: Int,
|
||||||
|
val outputHeight: Int,
|
||||||
|
) {
|
||||||
|
companion object {
|
||||||
|
/** Most samples averaged along each axis per output pixel; bounds work for big downscales. */
|
||||||
|
const val MAX_TAPS = 4
|
||||||
|
|
||||||
|
/** Snap any angle to the nearest of 0/90/180/270, the only rotations containers use. */
|
||||||
|
@JvmStatic
|
||||||
|
fun normalizeRotation(degrees: Int): Int = (((((degrees % 360) + 360) % 360 + 45) / 90) % 4) * 90
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Luma to display gray. Limited ("TV") range video codes black at 16 and white at 235, so
|
||||||
|
* it is stretched to 0..255 first. Monochrome thresholds at mid-gray to 0 or 255 (a
|
||||||
|
* 1-bit image compresses and delta-encodes far better than 16 levels); otherwise the
|
||||||
|
* gamma curve applies, as for photos (see GrayPacket.fromArgb).
|
||||||
|
*/
|
||||||
|
@JvmStatic
|
||||||
|
fun toneLut(limitedRange: Boolean, gamma: Float, monochrome: Boolean): ByteArray {
|
||||||
|
val curve = GrayPacket.toneCurve(gamma)
|
||||||
|
val lut = ByteArray(256)
|
||||||
|
for (y in 0 until 256) {
|
||||||
|
val full = if (limitedRange) ((y - 16) * 255 + 109) / 219 else y
|
||||||
|
val v = full.coerceIn(0, 255)
|
||||||
|
lut[y] = (if (monochrome) (if (v >= 128) 255 else 0) else curve[v]).toByte()
|
||||||
|
}
|
||||||
|
return lut
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample positions for each output index along one axis: [count] entries per output,
|
||||||
|
* evenly spaced inside that output's box of source pixels.
|
||||||
|
*/
|
||||||
|
private fun taps(source: Int, output: Int, count: Int): IntArray {
|
||||||
|
val positions = IntArray(output * count)
|
||||||
|
for (i in 0 until output) {
|
||||||
|
for (j in 0 until count) {
|
||||||
|
// Center of sub-box j of output i, in source pixels: (i + (j + 0.5)/count) * source / output.
|
||||||
|
val numerator = (i.toLong() * count + j) * 2 + 1
|
||||||
|
val position = (numerator * source / (2L * count * output)).toInt()
|
||||||
|
positions[i * count + j] = position.coerceIn(0, source - 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return positions
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val swapsAxes = rotation == 90 || rotation == 270
|
||||||
|
/** Scaled size before rotation (coded orientation). */
|
||||||
|
private val scaledWidth = if (swapsAxes) outputHeight else outputWidth
|
||||||
|
private val scaledHeight = if (swapsAxes) outputWidth else outputHeight
|
||||||
|
private val tapsX = ((sourceWidth + scaledWidth - 1) / scaledWidth).coerceIn(1, MAX_TAPS)
|
||||||
|
private val tapsY = ((sourceHeight + scaledHeight - 1) / scaledHeight).coerceIn(1, MAX_TAPS)
|
||||||
|
private val xs = taps(sourceWidth, scaledWidth, tapsX)
|
||||||
|
private val ys = taps(sourceHeight, scaledHeight, tapsY)
|
||||||
|
private val sums = IntArray(scaledWidth)
|
||||||
|
/** Unrotated scaled frame, needed only when a rotation follows. */
|
||||||
|
private val scaled: ByteArray? = if (rotation == 0) null else ByteArray(scaledWidth * scaledHeight)
|
||||||
|
|
||||||
|
init {
|
||||||
|
require(sourceWidth > 0 && sourceHeight > 0) { "bad source size ${sourceWidth}x$sourceHeight" }
|
||||||
|
require(outputWidth > 0 && outputHeight > 0) { "bad output size ${outputWidth}x$outputHeight" }
|
||||||
|
require(rotation == 0 || rotation == 90 || rotation == 180 || rotation == 270) { "bad rotation $rotation" }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scale the luma plane into [out] (outputWidth * outputHeight bytes, row-major). The visible
|
||||||
|
* pixel (x, y) is at luma[offset + y * rowStride + x * pixelStride]; [lut] is a 256-entry
|
||||||
|
* table from [toneLut].
|
||||||
|
*/
|
||||||
|
fun scale(luma: ByteArray, offset: Int, rowStride: Int, pixelStride: Int, lut: ByteArray, out: ByteArray) {
|
||||||
|
require(out.size >= outputWidth * outputHeight) { "output buffer too small" }
|
||||||
|
val target = scaled ?: out
|
||||||
|
val count = tapsX * tapsY
|
||||||
|
val half = count / 2
|
||||||
|
for (oy in 0 until scaledHeight) {
|
||||||
|
sums.fill(0)
|
||||||
|
for (ty in 0 until tapsY) {
|
||||||
|
val row = offset + ys[oy * tapsY + ty] * rowStride
|
||||||
|
var tap = 0
|
||||||
|
for (ox in 0 until scaledWidth) {
|
||||||
|
var sum = 0
|
||||||
|
for (tx in 0 until tapsX) {
|
||||||
|
sum += luma[row + xs[tap++] * pixelStride].toInt() and 0xff
|
||||||
|
}
|
||||||
|
sums[ox] += sum
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val base = oy * scaledWidth
|
||||||
|
for (ox in 0 until scaledWidth) {
|
||||||
|
target[base + ox] = lut[(sums[ox] + half) / count]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (scaled != null) rotate(scaled, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rotate the coded-orientation frame clockwise by [rotation] into display orientation. */
|
||||||
|
private fun rotate(source: ByteArray, out: ByteArray) {
|
||||||
|
val w = scaledWidth
|
||||||
|
val h = scaledHeight
|
||||||
|
for (y in 0 until outputHeight) {
|
||||||
|
val base = y * outputWidth
|
||||||
|
for (x in 0 until outputWidth) {
|
||||||
|
out[base + x] = when (rotation) {
|
||||||
|
90 -> source[(h - 1 - x) * w + y]
|
||||||
|
180 -> source[(h - 1 - y) * w + (w - 1 - x)]
|
||||||
|
else -> source[x * w + (w - 1 - y)]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -56,6 +56,7 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
|
|||||||
const load = loader({ global: { isIOS: true } }, {
|
const load = loader({ global: { isIOS: true } }, {
|
||||||
'../../native/file-access': {}, '../../native/font-files': { isFontFile: () => false },
|
'../../native/file-access': {}, '../../native/font-files': { isFontFile: () => false },
|
||||||
'../../graphics/installed-fonts': {}, '../../native/image-files': { isDecodableImageFile: () => false },
|
'../../graphics/installed-fonts': {}, '../../native/image-files': { isDecodableImageFile: () => false },
|
||||||
|
'../../native/video-player': { isPlayableVideoFile: () => true },
|
||||||
'../evenhub/installed-apps': { readEvenHubPackageManifest: () => null }, '../evenhub/permission-dialog': {},
|
'../evenhub/installed-apps': { readEvenHubPackageManifest: () => null }, '../evenhub/permission-dialog': {},
|
||||||
'./file-browser': { FileBrowserLayer: class { constructor(options) { browserOptions = options; } } },
|
'./file-browser': { FileBrowserLayer: class { constructor(options) { browserOptions = options; } } },
|
||||||
'./file-info-dialog': { FileInfoDialogLayer: class { constructor(entry, value) { actions = value; } } },
|
'./file-info-dialog': { FileInfoDialogLayer: class { constructor(entry, value) { actions = value; } } },
|
||||||
@@ -64,6 +65,8 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
|
|||||||
});
|
});
|
||||||
load('app/apps/files/files-app.ts').createFilesAppWindow({ openEhpkApp: path => { opened = path; }, installEhpkApp: path => { installed = path; } });
|
load('app/apps/files/files-app.ts').createFilesAppWindow({ openEhpkApp: path => { opened = path; }, installEhpkApp: path => { installed = path; } });
|
||||||
assert.equal(browserOptions.isSupportedFile('test.EHPK'), true);
|
assert.equal(browserOptions.isSupportedFile('test.EHPK'), true);
|
||||||
|
// Video playback is Android-only for now.
|
||||||
|
assert.equal(browserOptions.isSupportedFile('clip.mp4'), false);
|
||||||
const ctx = { stack: { push() {}, pop() {} } };
|
const ctx = { stack: { push() {}, pop() {} } };
|
||||||
browserOptions.onFilePicked({ name: 'test.ehpk', path: '/Documents/test.ehpk' }, ctx);
|
browserOptions.onFilePicked({ name: 'test.ehpk', path: '/Documents/test.ehpk' }, ctx);
|
||||||
assert.deepEqual(Array.from(actions, a => a.label), ['Run app', 'Install']);
|
assert.deepEqual(Array.from(actions, a => a.label), ['Run app', 'Install']);
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
import kotlin.test.*
|
||||||
|
|
||||||
|
class VideoFrameScalerTest {
|
||||||
|
private val identity = ByteArray(256) { it.toByte() }
|
||||||
|
|
||||||
|
private fun scale(scaler: VideoFrameScaler, luma: ByteArray, lut: ByteArray = identity): IntArray {
|
||||||
|
val out = ByteArray(scaler.outputWidth * scaler.outputHeight)
|
||||||
|
scaler.scale(luma, 0, scaler.sourceWidth, 1, lut, out)
|
||||||
|
return IntArray(out.size) { out[it].toInt() and 0xff }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun twoToOneDownscaleAveragesEachBox() {
|
||||||
|
// 4x2 source -> 2x1: each output is the mean of a 2x2 box.
|
||||||
|
val luma = byteArrayOf(0, 100, 200.toByte(), 200.toByte(), 20, 80, 100, 100)
|
||||||
|
val scaler = VideoFrameScaler(4, 2, 0, 2, 1)
|
||||||
|
assertContentEquals(intArrayOf(50, 150), scale(scaler, luma))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun largeDownscaleStaysWithinTheTapBudget() {
|
||||||
|
// 100:1 horizontally still averages only MAX_TAPS samples, spread across the box.
|
||||||
|
val luma = ByteArray(200) { (if (it % 100 < 50) 0 else 200).toByte() }
|
||||||
|
val scaler = VideoFrameScaler(200, 1, 0, 2, 1)
|
||||||
|
assertContentEquals(intArrayOf(100, 100), scale(scaler, luma))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun upscaleIsNearestNeighbor() {
|
||||||
|
val scaler = VideoFrameScaler(2, 1, 0, 4, 2)
|
||||||
|
assertContentEquals(intArrayOf(10, 10, 90, 90, 10, 10, 90, 90), scale(scaler, byteArrayOf(10, 90)))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun offsetAndStridesSelectTheVisiblePlane() {
|
||||||
|
// A 2x2 visible crop at (1, 1) of a 4-wide plane, with interleaved samples (pixelStride 2).
|
||||||
|
val plane = ByteArray(40)
|
||||||
|
fun put(x: Int, y: Int, v: Int) { plane[(1 + y) * 10 + 1 + x * 2] = v.toByte() }
|
||||||
|
put(0, 0, 11); put(1, 0, 22); put(0, 1, 33); put(1, 1, 44)
|
||||||
|
val scaler = VideoFrameScaler(2, 2, 0, 2, 2)
|
||||||
|
val out = ByteArray(4)
|
||||||
|
scaler.scale(plane, 11, 10, 2, identity, out)
|
||||||
|
assertContentEquals(byteArrayOf(11, 22, 33, 44), out)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun rotationsMatchTheContainerClockwiseConvention() {
|
||||||
|
// Coded 3x2: 1 2 3
|
||||||
|
// 4 5 6
|
||||||
|
val luma = byteArrayOf(1, 2, 3, 4, 5, 6)
|
||||||
|
assertContentEquals(intArrayOf(4, 1, 5, 2, 6, 3), scale(VideoFrameScaler(3, 2, 90, 2, 3), luma))
|
||||||
|
assertContentEquals(intArrayOf(6, 5, 4, 3, 2, 1), scale(VideoFrameScaler(3, 2, 180, 3, 2), luma))
|
||||||
|
assertContentEquals(intArrayOf(3, 6, 2, 5, 1, 4), scale(VideoFrameScaler(3, 2, 270, 2, 3), luma))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun normalizeRotationSnapsToQuarterTurns() {
|
||||||
|
assertEquals(0, VideoFrameScaler.normalizeRotation(0))
|
||||||
|
assertEquals(90, VideoFrameScaler.normalizeRotation(90))
|
||||||
|
assertEquals(270, VideoFrameScaler.normalizeRotation(-90))
|
||||||
|
assertEquals(180, VideoFrameScaler.normalizeRotation(540))
|
||||||
|
assertEquals(0, VideoFrameScaler.normalizeRotation(359))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun limitedRangeStretchesAndMonochromeThresholds() {
|
||||||
|
val gray = VideoFrameScaler.toneLut(limitedRange = true, gamma = 1f, monochrome = false)
|
||||||
|
assertEquals(0, gray[16].toInt() and 0xff)
|
||||||
|
assertEquals(0, gray[0].toInt() and 0xff)
|
||||||
|
assertEquals(255, gray[235].toInt() and 0xff)
|
||||||
|
assertEquals(255, gray[255].toInt() and 0xff)
|
||||||
|
val mono = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = true)
|
||||||
|
assertEquals(0, mono[127].toInt() and 0xff)
|
||||||
|
assertEquals(255, mono[128].toInt() and 0xff)
|
||||||
|
// Gamma darkens midtones but leaves the ends alone.
|
||||||
|
val photo = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = false)
|
||||||
|
assertTrue((photo[128].toInt() and 0xff) < 64)
|
||||||
|
assertEquals(255, photo[255].toInt() and 0xff)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,6 +36,7 @@
|
|||||||
"../app/g2/glasses-display-state.ts",
|
"../app/g2/glasses-display-state.ts",
|
||||||
"../app/g2/glance-state.ts",
|
"../app/g2/glance-state.ts",
|
||||||
"../app/apps/glanceboard/layout.ts",
|
"../app/apps/glanceboard/layout.ts",
|
||||||
|
"../app/apps/files/video-layout.ts",
|
||||||
"../app/g2/pairing-candidates.ts",
|
"../app/g2/pairing-candidates.ts",
|
||||||
"../app/apps/calculator/math/*.ts",
|
"../app/apps/calculator/math/*.ts",
|
||||||
"../app/apps/microphones/mic-protocol.ts",
|
"../app/apps/microphones/mic-protocol.ts",
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
const test = require("node:test");
|
||||||
|
const assert = require("node:assert/strict");
|
||||||
|
|
||||||
|
const {
|
||||||
|
fitVideo,
|
||||||
|
formatMediaTime,
|
||||||
|
videoHeightMode,
|
||||||
|
videoRect,
|
||||||
|
} = require("../.test-build/app/apps/files/video-layout.js");
|
||||||
|
|
||||||
|
const WINDOW = { width: 576, height: 260 };
|
||||||
|
const TALL_WINDOW = { width: 576, height: 452 };
|
||||||
|
// ../g2flash/demos/badapple.mp4
|
||||||
|
const BAD_APPLE = { width: 1444, height: 1080 };
|
||||||
|
|
||||||
|
test("fixed sizes fit their box and keep the aspect ratio", () => {
|
||||||
|
assert.deepEqual(videoRect("small", BAD_APPLE, WINDOW), { x: 208, y: 70, width: 160, height: 120 });
|
||||||
|
assert.deepEqual(videoRect("medium", BAD_APPLE, WINDOW), { x: 168, y: 40, width: 240, height: 180 });
|
||||||
|
const wide = videoRect("small", { width: 1920, height: 1080 }, WINDOW);
|
||||||
|
assert.equal(wide.width, 160);
|
||||||
|
assert.equal(wide.height, 90);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("large fills the window height; full screen fills the tall window's width", () => {
|
||||||
|
assert.deepEqual(videoRect("large", BAD_APPLE, WINDOW), { x: 112, y: 0, width: 348, height: 260 });
|
||||||
|
assert.deepEqual(videoRect("full", BAD_APPLE, TALL_WINDOW), { x: 0, y: 10, width: 576, height: 430 });
|
||||||
|
assert.equal(videoHeightMode("full"), "max");
|
||||||
|
assert.equal(videoHeightMode("large"), "min");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rects are aligned for the delta encoder and stay inside the viewport", () => {
|
||||||
|
for (const size of ["small", "medium", "large", "full"]) {
|
||||||
|
for (const video of [BAD_APPLE, { width: 1080, height: 1920 }, { width: 100, height: 20 }, { width: 7, height: 5 }]) {
|
||||||
|
for (const viewport of [WINDOW, TALL_WINDOW, { width: 150, height: 100 }]) {
|
||||||
|
const rect = videoRect(size, video, viewport);
|
||||||
|
const where = `${size} ${video.width}x${video.height} in ${viewport.width}x${viewport.height}`;
|
||||||
|
assert.equal(rect.x % 4, 0, where);
|
||||||
|
assert.equal(rect.y % 2, 0, where);
|
||||||
|
assert.equal(rect.width % 2, 0, where);
|
||||||
|
assert.equal(rect.height % 2, 0, where);
|
||||||
|
assert.ok(rect.width >= 2 && rect.height >= 2, where);
|
||||||
|
assert.ok(rect.x + rect.width <= viewport.width, where);
|
||||||
|
assert.ok(rect.y + rect.height <= viewport.height, where);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a tiny video is scaled up to the chosen size", () => {
|
||||||
|
assert.deepEqual(fitVideo({ width: 80, height: 60 }, { width: 240, height: 180 }), { width: 240, height: 180 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("media times", () => {
|
||||||
|
assert.equal(formatMediaTime(0), "0:00");
|
||||||
|
assert.equal(formatMediaTime(232_130), "3:52");
|
||||||
|
assert.equal(formatMediaTime(3_723_000), "1:02:03");
|
||||||
|
});
|
||||||
Reference in new issue
Block a user