forked from Blendman974/faceclaw
Add a video player, launched by selecting a video file from the Files app
This commit is contained in:
1 parent
deaa93fead
commit
cb6dddd120
13 files changed
+1784
-3
No files matched your search
@@ -0,0 +1,791 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFocusRequest
|
||||
import android.media.AudioManager
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaCodecInfo
|
||||
import android.media.MediaExtractor
|
||||
import android.media.MediaFormat
|
||||
import android.media.MediaPlayer
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
|
||||
import org.json.JSONObject
|
||||
|
||||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import java.util.zip.CRC32
|
||||
import kotlin.concurrent.withLock
|
||||
|
||||
/**
|
||||
* Plays a video file onto the glasses. A background thread decodes the video track
|
||||
* (MediaExtractor + MediaCodec), scales each due frame's luma to gray (VideoFrameScaler) and
|
||||
* submits it straight into a rect of a window's compositor surface, through the same
|
||||
* submitSurfaceFrame entry point app workers use, so no JS runs per frame. When wanted, the
|
||||
* audio track plays through a MediaPlayer on the phone's default media route, and the video
|
||||
* clock follows its position.
|
||||
*
|
||||
* The TS side (app/native/video-player.ts) owns the window: it paints the rest of the surface
|
||||
* and repaints the video rect from [copyLastFrame] whenever it renders. Frames are submitted
|
||||
* only while playing, and [pause] returns only once no further frame can be submitted, so a
|
||||
* repaint right after it (the paused overlay) is never overwritten.
|
||||
*
|
||||
* Control methods are for the main thread; listener callbacks are posted to it.
|
||||
*/
|
||||
class FaceclawVideoPlayer(context: Context) {
|
||||
interface Listener {
|
||||
/**
|
||||
* "playing", "paused" or "ended", with the media position. Also sent as "paused" when a
|
||||
* seek made while paused has decoded its new frame (see [copyLastFrame]).
|
||||
*/
|
||||
fun onStateChanged(state: String?, positionMs: Long)
|
||||
|
||||
/** Playback stopped on an error (unreadable file, no decoder, ...). */
|
||||
fun onError(message: String?)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "FaceclawVideo"
|
||||
private const val CODEC_TIMEOUT_US = 10_000L
|
||||
/** Longest single sleep while waiting for a frame to come due, so commands stay responsive. */
|
||||
private const val MAX_WAIT_US = 20_000L
|
||||
/** A frame this far behind the clock is dropped undrawn... */
|
||||
private const val LATE_DROP_US = 50_000L
|
||||
/** ...unless nothing has been drawn for this long, so a slow phone still shows progress. */
|
||||
private const val MAX_DROP_GAP_NANOS = 250_000_000L
|
||||
/**
|
||||
* Decoded frames closer together than this are thinned: past ~30fps the glasses link
|
||||
* could never show them, and each one still costs a scale and a composite.
|
||||
*/
|
||||
private const val MIN_FRAME_INTERVAL_US = 30_000L
|
||||
/** Audio position is ignored this long after the clock starts (output startup latency). */
|
||||
private const val AUDIO_SETTLE_NANOS = 500_000_000L
|
||||
/** Clock drift from the audio position that makes the video clock jump to it. */
|
||||
private const val AUDIO_RESYNC_US = 60_000L
|
||||
/** How often the audio position is read (a binder call into the media server). */
|
||||
private const val AUDIO_CHECK_INTERVAL_NANOS = 250_000_000L
|
||||
/** A seek never lands closer than this to the end, so it always shows a frame. */
|
||||
private const val SEEK_END_MARGIN_US = 500_000L
|
||||
|
||||
private val AUDIO_ATTRIBUTES: AudioAttributes = AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE)
|
||||
.build()
|
||||
|
||||
private fun MediaFormat.intOr(key: String, fallback: Int): Int =
|
||||
if (containsKey(key)) getInteger(key) else fallback
|
||||
|
||||
private fun MediaFormat.longOr(key: String, fallback: Long): Long =
|
||||
if (containsKey(key)) getLong(key) else fallback
|
||||
|
||||
private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int {
|
||||
for (i in 0 until extractor.trackCount) {
|
||||
val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue
|
||||
if (mime.startsWith(mimePrefix)) return i
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
/**
|
||||
* Describe a video file for the settings panel, as JSON: {width, height} in display
|
||||
* orientation (rotation applied), rotation, durationMs, hasAudio, mime; or {error}.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun probe(path: String?): String {
|
||||
val result = JSONObject()
|
||||
val extractor = MediaExtractor()
|
||||
try {
|
||||
extractor.setDataSource(path ?: throw IOException("no path"))
|
||||
val videoTrack = findTrack(extractor, "video/")
|
||||
if (videoTrack < 0) {
|
||||
result.put("error", "No video track")
|
||||
return result.toString()
|
||||
}
|
||||
var durationUs = 0L
|
||||
for (i in 0 until extractor.trackCount) {
|
||||
durationUs = Math.max(durationUs, extractor.getTrackFormat(i).longOr(MediaFormat.KEY_DURATION, 0L))
|
||||
}
|
||||
val format = extractor.getTrackFormat(videoTrack)
|
||||
val rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
|
||||
val codedWidth = format.intOr(MediaFormat.KEY_WIDTH, 0)
|
||||
val codedHeight = format.intOr(MediaFormat.KEY_HEIGHT, 0)
|
||||
val swap = rotation == 90 || rotation == 270
|
||||
result.put("width", if (swap) codedHeight else codedWidth)
|
||||
result.put("height", if (swap) codedWidth else codedHeight)
|
||||
result.put("rotation", rotation)
|
||||
result.put("durationMs", durationUs / 1000)
|
||||
result.put("hasAudio", findTrack(extractor, "audio/") >= 0)
|
||||
result.put("mime", format.getString(MediaFormat.KEY_MIME) ?: "")
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "probe failed for $path", e)
|
||||
result.put("error", e.message ?: e.toString())
|
||||
} finally {
|
||||
extractor.release()
|
||||
}
|
||||
return result.toString()
|
||||
}
|
||||
}
|
||||
|
||||
/** What one start() call asked for; fixed for that playback's lifetime. */
|
||||
private class Params(
|
||||
val path: String,
|
||||
val surfaceId: String,
|
||||
val x: Int,
|
||||
val y: Int,
|
||||
val width: Int,
|
||||
val height: Int,
|
||||
val monochrome: Boolean,
|
||||
val gamma: Float,
|
||||
val audio: Boolean,
|
||||
)
|
||||
|
||||
private enum class Wait { DUE, INTERRUPTED, STOP }
|
||||
|
||||
private val appContext: Context = context.applicationContext
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
@Volatile private var listener: Listener? = null
|
||||
|
||||
private val lock = ReentrantLock()
|
||||
private val wake = lock.newCondition()
|
||||
|
||||
// Everything below is guarded by lock.
|
||||
/** Bumped by every start/stop; a playback thread whose generation is stale exits. */
|
||||
private var generation = 0
|
||||
private var params: Params? = null
|
||||
/** The user's intent: playing or paused. */
|
||||
private var playing = false
|
||||
private var ended = false
|
||||
/** A seek (or the initial start) is decoding toward its target; the clock waits for its frame. */
|
||||
private var seeking = false
|
||||
private var pendingSeekUs = -1L
|
||||
private var durationUs = 0L
|
||||
private var clockRunning = false
|
||||
/** Media time while the clock is stopped. */
|
||||
private var heldUs = 0L
|
||||
private var clockBaseUs = 0L
|
||||
private var clockBaseNanos = 0L
|
||||
private var clockStartedNanos = 0L
|
||||
private var lastAudioCheckNanos = 0L
|
||||
private var audio: MediaPlayer? = null
|
||||
private var hasAudioFocus = false
|
||||
private var focusRequest: AudioFocusRequest? = null
|
||||
/** The most recently decoded frame (params.width x params.height), shown or previewed. */
|
||||
private var lastFrame: ByteArray? = null
|
||||
/** Checksum of the frame last submitted, so an unchanged frame is not recomposited; -1 = none. */
|
||||
private var lastSubmittedChecksum = -1L
|
||||
private var submitFailures = 0
|
||||
|
||||
private val focusListener = AudioManager.OnAudioFocusChangeListener { change ->
|
||||
when (change) {
|
||||
AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> {
|
||||
if (change == AudioManager.AUDIOFOCUS_LOSS) lock.withLock { hasAudioFocus = false }
|
||||
// Another app (a call, a music player) took the audio: pause like any player would.
|
||||
if (isPlaying()) {
|
||||
pause()
|
||||
notifyState(generationNow(), "paused", getPositionMs())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun setListener(listener: Listener?) {
|
||||
this.listener = listener
|
||||
}
|
||||
|
||||
/**
|
||||
* Start playing [path] from [startMs] into the (rectX, rectY, width, height) rect of
|
||||
* compositor surface [surfaceId], replacing any current playback. Decoding and audio setup
|
||||
* happen on the playback thread; failures arrive as [Listener.onError].
|
||||
*/
|
||||
fun start(
|
||||
path: String,
|
||||
surfaceId: String,
|
||||
rectX: Int,
|
||||
rectY: Int,
|
||||
width: Int,
|
||||
height: Int,
|
||||
monochrome: Boolean,
|
||||
gamma: Float,
|
||||
withAudio: Boolean,
|
||||
startMs: Long,
|
||||
) {
|
||||
require(width > 0 && height > 0) { "bad video rect ${width}x$height" }
|
||||
val p = Params(path, surfaceId, rectX, rectY, width, height, monochrome, gamma, withAudio)
|
||||
val gen = lock.withLock {
|
||||
endPlaybackLocked()
|
||||
params = p
|
||||
playing = true
|
||||
ended = false
|
||||
seeking = true
|
||||
pendingSeekUs = Math.max(0L, startMs) * 1000
|
||||
heldUs = pendingSeekUs
|
||||
// The old frame may not match the new settings (size, monochrome).
|
||||
lastFrame = null
|
||||
lastSubmittedChecksum = -1
|
||||
submitFailures = 0
|
||||
generation
|
||||
}
|
||||
val thread = Thread({ Playback(gen, p).run() }, "FaceclawVideo")
|
||||
thread.priority = Thread.NORM_PRIORITY + 1
|
||||
thread.start()
|
||||
}
|
||||
|
||||
/** Pause; once this returns no further frame is submitted until [resume]. */
|
||||
fun pause() {
|
||||
lock.withLock {
|
||||
if (!playing) return
|
||||
playing = false
|
||||
holdClockLocked(clockUsLocked())
|
||||
wake.signalAll()
|
||||
}
|
||||
}
|
||||
|
||||
fun resume() {
|
||||
lock.withLock {
|
||||
if (playing || ended || params == null) return
|
||||
playing = true
|
||||
// A fresh frame must go out even if it matches the last one: the TS side painted over
|
||||
// the surface while paused.
|
||||
lastSubmittedChecksum = -1
|
||||
if (!seeking) startClockLocked()
|
||||
wake.signalAll()
|
||||
}
|
||||
}
|
||||
|
||||
/** Seek to [positionMs]; while paused, the new frame is decoded and announced as "paused". */
|
||||
fun seekTo(positionMs: Long) {
|
||||
lock.withLock {
|
||||
if (params == null || ended) return
|
||||
var targetUs = Math.max(0L, positionMs) * 1000
|
||||
if (durationUs > 0) targetUs = Math.min(targetUs, Math.max(0L, durationUs - SEEK_END_MARGIN_US))
|
||||
pendingSeekUs = targetUs
|
||||
seeking = true
|
||||
holdClockLocked(targetUs)
|
||||
lastSubmittedChecksum = -1
|
||||
wake.signalAll()
|
||||
}
|
||||
}
|
||||
|
||||
fun getPositionMs(): Long = lock.withLock { clockUsLocked() / 1000 }
|
||||
|
||||
fun getDurationMs(): Long = lock.withLock { durationUs / 1000 }
|
||||
|
||||
fun isPlaying(): Boolean = lock.withLock { playing && !ended && params != null }
|
||||
|
||||
/**
|
||||
* Copy the most recent frame into [out], which must hold exactly the video rect's
|
||||
* width*height bytes (row-major 8bpp gray). False when there is no frame of that size yet.
|
||||
*/
|
||||
fun copyLastFrame(out: ByteBuffer): Boolean {
|
||||
lock.withLock {
|
||||
val frame = lastFrame ?: return false
|
||||
if (out.capacity() != frame.size) return false
|
||||
out.clear()
|
||||
out.put(frame)
|
||||
out.clear()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/** End playback and free the decoder and audio; the last frame stays readable. */
|
||||
fun stop() {
|
||||
lock.withLock { endPlaybackLocked() }
|
||||
}
|
||||
|
||||
/** stop(), and drop the listener and frame. */
|
||||
fun release() {
|
||||
lock.withLock {
|
||||
endPlaybackLocked()
|
||||
lastFrame = null
|
||||
}
|
||||
listener = null
|
||||
}
|
||||
|
||||
private fun generationNow(): Int = lock.withLock { generation }
|
||||
|
||||
private fun endPlaybackLocked() {
|
||||
generation++
|
||||
params = null
|
||||
playing = false
|
||||
seeking = false
|
||||
pendingSeekUs = -1
|
||||
// Keep the position readable (an error's caller resumes from it).
|
||||
heldUs = clockUsLocked()
|
||||
clockRunning = false
|
||||
releaseAudioLocked()
|
||||
wake.signalAll()
|
||||
}
|
||||
|
||||
// ---- clock (all under lock) ----
|
||||
|
||||
private fun clockUsLocked(): Long =
|
||||
if (!clockRunning) heldUs else clockBaseUs + (System.nanoTime() - clockBaseNanos) / 1000
|
||||
|
||||
private fun startClockLocked() {
|
||||
if (clockRunning) return
|
||||
clockRunning = true
|
||||
clockBaseUs = heldUs
|
||||
clockBaseNanos = System.nanoTime()
|
||||
clockStartedNanos = clockBaseNanos
|
||||
val player = audio ?: return
|
||||
requestAudioFocusLocked()
|
||||
try {
|
||||
player.start()
|
||||
} catch (e: IllegalStateException) {
|
||||
Log.w(TAG, "audio start failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun holdClockLocked(atUs: Long) {
|
||||
heldUs = atUs
|
||||
if (!clockRunning) return
|
||||
clockRunning = false
|
||||
try {
|
||||
audio?.let { if (it.isPlaying) it.pause() }
|
||||
} catch (e: IllegalStateException) {
|
||||
Log.w(TAG, "audio pause failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the video clock on the audio position: output latency and seeks make the two drift,
|
||||
* and a picture ahead of or behind its sound is the one sync error people notice.
|
||||
*/
|
||||
private fun followAudioLocked() {
|
||||
val player = audio ?: return
|
||||
val now = System.nanoTime()
|
||||
if (!clockRunning || now - clockStartedNanos < AUDIO_SETTLE_NANOS) return
|
||||
if (now - lastAudioCheckNanos < AUDIO_CHECK_INTERVAL_NANOS) return
|
||||
lastAudioCheckNanos = now
|
||||
val audioUs = try {
|
||||
if (!player.isPlaying) return // finished (audio shorter than video) or not started
|
||||
player.currentPosition * 1000L
|
||||
} catch (e: IllegalStateException) {
|
||||
return
|
||||
}
|
||||
if (Math.abs(audioUs - clockUsLocked()) > AUDIO_RESYNC_US) {
|
||||
clockBaseUs = audioUs
|
||||
clockBaseNanos = System.nanoTime()
|
||||
}
|
||||
}
|
||||
|
||||
private fun seekAudioLocked(targetUs: Long) {
|
||||
val player = audio ?: return
|
||||
try {
|
||||
if (Build.VERSION.SDK_INT >= 26) {
|
||||
player.seekTo(targetUs / 1000, MediaPlayer.SEEK_CLOSEST)
|
||||
} else {
|
||||
player.seekTo((targetUs / 1000).toInt())
|
||||
}
|
||||
} catch (e: IllegalStateException) {
|
||||
Log.w(TAG, "audio seek failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun requestAudioFocusLocked() {
|
||||
if (hasAudioFocus) return
|
||||
val result = if (Build.VERSION.SDK_INT >= 26) {
|
||||
val request = focusRequest ?: AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN)
|
||||
.setAudioAttributes(AUDIO_ATTRIBUTES)
|
||||
.setOnAudioFocusChangeListener(focusListener, mainHandler)
|
||||
.build()
|
||||
.also { focusRequest = it }
|
||||
audioManager.requestAudioFocus(request)
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
audioManager.requestAudioFocus(focusListener, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN)
|
||||
}
|
||||
// Play regardless: the sound was asked for, and a refusal is transient (a call ending).
|
||||
hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED
|
||||
}
|
||||
|
||||
private fun releaseAudioLocked() {
|
||||
val player = audio ?: return
|
||||
audio = null
|
||||
try {
|
||||
player.release()
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "audio release failed", e)
|
||||
}
|
||||
if (hasAudioFocus) {
|
||||
hasAudioFocus = false
|
||||
if (Build.VERSION.SDK_INT >= 26) {
|
||||
focusRequest?.let { audioManager.abandonAudioFocusRequest(it) }
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
audioManager.abandonAudioFocus(focusListener)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- notifications ----
|
||||
|
||||
private fun notifyState(gen: Int, state: String, positionMs: Long) {
|
||||
mainHandler.post {
|
||||
if (lock.withLock { gen == generation }) listener?.onStateChanged(state, positionMs)
|
||||
}
|
||||
}
|
||||
|
||||
private fun notifyError(gen: Int, message: String) {
|
||||
mainHandler.post {
|
||||
if (lock.withLock { gen == generation }) listener?.onError(message)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- frame output (under lock) ----
|
||||
|
||||
private fun submitLocked(p: Params, pixels: ByteArray, checksum: Long) {
|
||||
val buffer = ByteBuffer.wrap(pixels)
|
||||
// Frame id 0: per-frame FrameTimings entries at video rate would crowd every real
|
||||
// input/render frame out of the timing export.
|
||||
val fingerprint = "video:$generation:" + java.lang.Long.toHexString(checksum)
|
||||
try {
|
||||
val communicator = FaceclawBleCommunicator.getActive()
|
||||
if (communicator != null) {
|
||||
communicator.submitSurfaceFrame(buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
|
||||
} else {
|
||||
FaceclawPreviewCompositor.getActive()?.submitSurfaceFrame(
|
||||
buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null)
|
||||
}
|
||||
submitFailures = 0
|
||||
} catch (e: RuntimeException) {
|
||||
// The window's surface went away or shrank under us (closed, relaid out): drop the frame.
|
||||
if (submitFailures++ == 0) Log.w(TAG, "video frame submit failed: ${e.message}")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One start() call's decode thread. Owns the extractor and decoder; everything shared with
|
||||
* the control methods goes through the player's lock.
|
||||
*/
|
||||
private inner class Playback(private val gen: Int, private val p: Params) {
|
||||
private var extractor: MediaExtractor? = null
|
||||
private var codec: MediaCodec? = null
|
||||
private val info = MediaCodec.BufferInfo()
|
||||
private var inputDone = false
|
||||
private var outputDone = false
|
||||
private var trackEndUs = 0L
|
||||
/** Frames before this are decoded only on the way to a seek target. */
|
||||
private var skipUntilUs = 0L
|
||||
private var lastDrawnPtsUs = Long.MIN_VALUE / 2
|
||||
private var lastDrawnNanos = 0L
|
||||
private var rotation = 0
|
||||
private var lut = ByteArray(256)
|
||||
private var scaler: VideoFrameScaler? = null
|
||||
private var luma = ByteArray(0)
|
||||
/** The frame being built; swapped with lastFrame when shown. */
|
||||
private var frame = ByteArray(p.width * p.height)
|
||||
private val crc = CRC32()
|
||||
|
||||
fun run() {
|
||||
try {
|
||||
open()
|
||||
loop()
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "playback failed for ${p.path}", t)
|
||||
val current = lock.withLock {
|
||||
val current = gen == generation
|
||||
if (current) endPlaybackLocked()
|
||||
current
|
||||
}
|
||||
// endPlaybackLocked bumped the generation; report under the new one.
|
||||
if (current) notifyError(generationNow(), t.message ?: t.toString())
|
||||
} finally {
|
||||
try {
|
||||
codec?.stop()
|
||||
} catch (ignored: Throwable) {
|
||||
}
|
||||
codec?.release()
|
||||
extractor?.release()
|
||||
}
|
||||
}
|
||||
|
||||
private fun current(): Boolean = lock.withLock { gen == generation }
|
||||
|
||||
private fun open() {
|
||||
val ex = MediaExtractor()
|
||||
extractor = ex
|
||||
ex.setDataSource(p.path)
|
||||
val track = findTrack(ex, "video/")
|
||||
if (track < 0) throw IOException("No video track")
|
||||
ex.selectTrack(track)
|
||||
val format = ex.getTrackFormat(track)
|
||||
rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0))
|
||||
trackEndUs = format.longOr(MediaFormat.KEY_DURATION, 0L)
|
||||
// Most video is limited ("TV") range; only an explicit full-range tag says otherwise.
|
||||
val limited = format.intOr(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) !=
|
||||
MediaFormat.COLOR_RANGE_FULL
|
||||
lut = VideoFrameScaler.toneLut(limited, p.gamma, p.monochrome)
|
||||
val mime = format.getString(MediaFormat.KEY_MIME) ?: throw IOException("Unknown video format")
|
||||
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
|
||||
val decoder = try {
|
||||
MediaCodec.createDecoderByType(mime)
|
||||
} catch (e: Exception) {
|
||||
throw IOException("No decoder for $mime")
|
||||
}
|
||||
codec = decoder
|
||||
decoder.configure(format, null, null, 0)
|
||||
decoder.start()
|
||||
val player = if (p.audio && findTrack(ex, "audio/") >= 0) openAudio() else null
|
||||
lock.withLock {
|
||||
if (gen != generation) {
|
||||
player?.release()
|
||||
return
|
||||
}
|
||||
durationUs = trackEndUs
|
||||
// Positioned by the start seek, which is the first command loop() handles.
|
||||
audio = player
|
||||
}
|
||||
}
|
||||
|
||||
/** A prepared MediaPlayer for the file's audio, or null (played silent) if it won't open. */
|
||||
private fun openAudio(): MediaPlayer? {
|
||||
val player = MediaPlayer()
|
||||
return try {
|
||||
player.setAudioAttributes(AUDIO_ATTRIBUTES)
|
||||
player.setDataSource(p.path)
|
||||
player.prepare()
|
||||
player
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "audio unavailable for ${p.path}; playing silent", e)
|
||||
player.release()
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun loop() {
|
||||
while (true) {
|
||||
// Commands: stop, seek, or wait out a pause.
|
||||
var seekUs = -1L
|
||||
lock.withLock {
|
||||
while (true) {
|
||||
if (gen != generation) return
|
||||
if (pendingSeekUs >= 0) {
|
||||
seekUs = pendingSeekUs
|
||||
pendingSeekUs = -1
|
||||
break
|
||||
}
|
||||
if (playing || seeking) break
|
||||
wake.await()
|
||||
}
|
||||
}
|
||||
if (seekUs >= 0) {
|
||||
seek(seekUs)
|
||||
continue
|
||||
}
|
||||
if (outputDone) {
|
||||
if (awaitEnd()) return
|
||||
continue
|
||||
}
|
||||
feedInput()
|
||||
val index = codec!!.dequeueOutputBuffer(info, CODEC_TIMEOUT_US)
|
||||
if (index < 0) continue // try again later / format or buffers changed
|
||||
val endOfStream = (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0
|
||||
if (info.size > 0) {
|
||||
handleFrame(index, info.presentationTimeUs)
|
||||
} else {
|
||||
codec!!.releaseOutputBuffer(index, false)
|
||||
}
|
||||
if (endOfStream) outputDone = true
|
||||
}
|
||||
}
|
||||
|
||||
private fun seek(targetUs: Long) {
|
||||
lock.withLock {
|
||||
if (gen != generation) return
|
||||
seekAudioLocked(targetUs)
|
||||
}
|
||||
extractor!!.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC)
|
||||
codec!!.flush()
|
||||
inputDone = false
|
||||
outputDone = false
|
||||
skipUntilUs = targetUs
|
||||
lastDrawnPtsUs = Long.MIN_VALUE / 2
|
||||
}
|
||||
|
||||
private fun feedInput() {
|
||||
if (inputDone) return
|
||||
val decoder = codec!!
|
||||
val ex = extractor!!
|
||||
while (true) {
|
||||
val index = decoder.dequeueInputBuffer(0)
|
||||
if (index < 0) return
|
||||
val buffer = decoder.getInputBuffer(index) ?: return
|
||||
val size = ex.readSampleData(buffer, 0)
|
||||
if (size < 0) {
|
||||
decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
|
||||
inputDone = true
|
||||
return
|
||||
}
|
||||
decoder.queueInputBuffer(index, 0, size, ex.sampleTime, 0)
|
||||
ex.advance()
|
||||
}
|
||||
}
|
||||
|
||||
/** Decide a decoded frame's fate (skip, preview, drop, draw); always releases it. */
|
||||
private fun handleFrame(index: Int, ptsUs: Long) {
|
||||
try {
|
||||
if (ptsUs < skipUntilUs) return
|
||||
val landingSeek = lock.withLock { seeking }
|
||||
if (landingSeek) {
|
||||
landSeek(index, ptsUs)
|
||||
return
|
||||
}
|
||||
if (ptsUs - lastDrawnPtsUs < MIN_FRAME_INTERVAL_US) return
|
||||
if (waitUntilDue(ptsUs) != Wait.DUE) return
|
||||
val lateUs = lock.withLock { clockUsLocked() } - ptsUs
|
||||
if (lateUs > LATE_DROP_US && System.nanoTime() - lastDrawnNanos < MAX_DROP_GAP_NANOS) return
|
||||
val checksum = convert(index)
|
||||
present(ptsUs, checksum)
|
||||
} finally {
|
||||
codec!!.releaseOutputBuffer(index, false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first frame at or past a seek target (or the start position): drawn and the clock
|
||||
* started when playing, kept as the paused preview otherwise.
|
||||
*/
|
||||
private fun landSeek(index: Int, ptsUs: Long) {
|
||||
val checksum = convert(index)
|
||||
var previewed = false
|
||||
lock.withLock {
|
||||
if (gen != generation || pendingSeekUs >= 0) return
|
||||
seeking = false
|
||||
heldUs = ptsUs
|
||||
swapInFrameLocked()
|
||||
if (playing) {
|
||||
submitLocked(p, lastFrame!!, checksum)
|
||||
lastSubmittedChecksum = checksum
|
||||
startClockLocked()
|
||||
} else {
|
||||
previewed = true
|
||||
}
|
||||
}
|
||||
lastDrawnPtsUs = ptsUs
|
||||
lastDrawnNanos = System.nanoTime()
|
||||
notifyState(gen, if (previewed) "paused" else "playing", ptsUs / 1000)
|
||||
}
|
||||
|
||||
/** Show a due frame, unless a pause or seek got in first. */
|
||||
private fun present(ptsUs: Long, checksum: Long) {
|
||||
lock.withLock {
|
||||
if (gen != generation || !playing || pendingSeekUs >= 0) return
|
||||
swapInFrameLocked()
|
||||
if (checksum != lastSubmittedChecksum) {
|
||||
submitLocked(p, lastFrame!!, checksum)
|
||||
lastSubmittedChecksum = checksum
|
||||
}
|
||||
}
|
||||
lastDrawnPtsUs = ptsUs
|
||||
lastDrawnNanos = System.nanoTime()
|
||||
}
|
||||
|
||||
private fun swapInFrameLocked() {
|
||||
val shown = frame
|
||||
val previous = lastFrame
|
||||
frame = if (previous != null && previous.size == shown.size) previous else ByteArray(shown.size)
|
||||
lastFrame = shown
|
||||
}
|
||||
|
||||
/** Block until the clock reaches [ptsUs]; paused time doesn't count. */
|
||||
private fun waitUntilDue(ptsUs: Long): Wait {
|
||||
lock.withLock {
|
||||
while (true) {
|
||||
if (gen != generation) return Wait.STOP
|
||||
if (pendingSeekUs >= 0) return Wait.INTERRUPTED
|
||||
if (!playing) {
|
||||
wake.await()
|
||||
continue
|
||||
}
|
||||
followAudioLocked()
|
||||
val now = clockUsLocked()
|
||||
if (now >= ptsUs) return Wait.DUE
|
||||
wake.await(Math.min(ptsUs - now, MAX_WAIT_US), TimeUnit.MICROSECONDS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The decoder is drained: hold the last frame until the clock reaches the end, then
|
||||
* report "ended". False when a seek interrupted the wait (playback continues).
|
||||
*/
|
||||
private fun awaitEnd(): Boolean {
|
||||
val endUs = Math.max(trackEndUs, lastDrawnPtsUs)
|
||||
val landing = lock.withLock { seeking }
|
||||
if (!landing && waitUntilDue(endUs) != Wait.DUE) return !current()
|
||||
lock.withLock {
|
||||
if (gen != generation) return true
|
||||
if (pendingSeekUs >= 0) return false
|
||||
playing = false
|
||||
seeking = false
|
||||
ended = true
|
||||
holdClockLocked(endUs)
|
||||
releaseAudioLocked()
|
||||
}
|
||||
notifyState(gen, "ended", endUs / 1000)
|
||||
return true
|
||||
}
|
||||
|
||||
/** Scale the output buffer's frame into [frame]; returns its checksum. */
|
||||
private fun convert(index: Int): Long {
|
||||
val decoder = codec!!
|
||||
val image = try {
|
||||
decoder.getOutputImage(index)
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
if (image != null) {
|
||||
try {
|
||||
val plane = image.planes[0]
|
||||
val crop = image.cropRect
|
||||
val offset = copyLuma(plane.buffer) + crop.top * plane.rowStride + crop.left * plane.pixelStride
|
||||
scaleInto(crop.width(), crop.height(), offset, plane.rowStride, plane.pixelStride)
|
||||
} finally {
|
||||
image.close()
|
||||
}
|
||||
} else {
|
||||
// No Image access: every YUV420 layout starts with the Y plane at the buffer start.
|
||||
val buffer = decoder.getOutputBuffer(index) ?: throw IOException("Decoder returned no frame")
|
||||
buffer.position(info.offset)
|
||||
buffer.limit(info.offset + info.size)
|
||||
val format = decoder.outputFormat
|
||||
val width = format.getInteger(MediaFormat.KEY_WIDTH)
|
||||
val stride = format.intOr(MediaFormat.KEY_STRIDE, width)
|
||||
val left = format.intOr("crop-left", 0)
|
||||
val top = format.intOr("crop-top", 0)
|
||||
val right = format.intOr("crop-right", width - 1)
|
||||
val bottom = format.intOr("crop-bottom", format.getInteger(MediaFormat.KEY_HEIGHT) - 1)
|
||||
copyLuma(buffer)
|
||||
scaleInto(right - left + 1, bottom - top + 1, top * stride + left, stride, 1)
|
||||
}
|
||||
crc.reset()
|
||||
crc.update(frame, 0, frame.size)
|
||||
return crc.value
|
||||
}
|
||||
|
||||
/** Bulk-copy the plane (one memcpy; per-byte reads from a direct buffer are slow). Returns 0. */
|
||||
private fun copyLuma(buffer: ByteBuffer): Int {
|
||||
val bytes = buffer.remaining()
|
||||
if (luma.size < bytes) luma = ByteArray(bytes)
|
||||
buffer.get(luma, 0, bytes)
|
||||
return 0
|
||||
}
|
||||
|
||||
private fun scaleInto(width: Int, height: Int, offset: Int, rowStride: Int, pixelStride: Int) {
|
||||
var s = scaler
|
||||
if (s == null || s.sourceWidth != width || s.sourceHeight != height) {
|
||||
s = VideoFrameScaler(width, height, rotation, p.width, p.height)
|
||||
scaler = s
|
||||
}
|
||||
s.scale(luma, offset, rowStride, pixelStride, lut, frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { isPlayableVideoFile } from "../../native/video-player";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
import { FileBrowserLayer } from "./file-browser";
|
||||
@@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
installEhpkApp: (path: string) => Promise<void> | void;
|
||||
/** Open a font file's previewer as its own shell window. */
|
||||
openFontWindow: (title: string, path: string) => void;
|
||||
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||
openVideoWindow: (title: string, path: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The Files app's launcher-opened window: a file browser over Places
|
||||
* (bookmarks and storage roots). Picking any file opens an info dialog with
|
||||
* metadata plus, for viewable types (text, images), the open actions.
|
||||
* metadata plus, for viewable types (text, images, video), the open actions.
|
||||
*/
|
||||
export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow {
|
||||
let created: InProcessWindow | null = null;
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)),
|
||||
isSupportedFile: (name) =>
|
||||
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isPlayableVideoFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
label: "Play video",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openVideoWindow(entry.name, entry.path);
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isFontFile(entry.name)) {
|
||||
// Install copies into the app-internal fonts directory, making the face
|
||||
// available in the Settings font pickers. The row relabels itself with
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
FILES_SURFACE_ID,
|
||||
FILES_WINDOW_ID,
|
||||
} from "./files-app";
|
||||
import { createVideoPlayerWindow } from "./video-window";
|
||||
|
||||
// Document windows are closeable and non-singleton; the serial keeps their
|
||||
// windowIds unique.
|
||||
@@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v
|
||||
});
|
||||
}
|
||||
|
||||
function openVideoWindow(ctx: AppContext, title: string, path: string): void {
|
||||
const windowId = `files:doc:${nextDocumentSerial++}`;
|
||||
void ctx
|
||||
.launchInProcessApp(windowId, `window:${windowId}`, (options) =>
|
||||
createVideoPlayerWindow(windowId, title, path, options),
|
||||
)
|
||||
.catch((error) => {
|
||||
ctx.appendLog(`video window failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
const filesApp: AppDefinition = {
|
||||
appId: "files",
|
||||
title: "Files",
|
||||
@@ -78,6 +90,7 @@ const filesApp: AppDefinition = {
|
||||
}
|
||||
},
|
||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||
}),
|
||||
),
|
||||
// A document arriving via Android's Share intent opens as its own window.
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Where a video goes in the player window, for each size choice. Pure
|
||||
* geometry, so it is unit-tested without the NativeScript runtime.
|
||||
*/
|
||||
|
||||
export type VideoSize = "small" | "medium" | "large" | "full";
|
||||
|
||||
export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"];
|
||||
|
||||
export type Size = { width: number; height: number };
|
||||
export type Rect = Size & { x: number; y: number };
|
||||
|
||||
/**
|
||||
* Bounding boxes for the fixed sizes; a video is fitted inside, keeping its
|
||||
* aspect. Large and full fit the whole viewport instead (full in a tall
|
||||
* window). Bigger pictures cost the glasses link more per frame, so smaller
|
||||
* sizes play at a higher frame rate.
|
||||
*/
|
||||
const FIXED_BOXES: Partial<Record<VideoSize, Size>> = {
|
||||
small: { width: 160, height: 120 },
|
||||
medium: { width: 240, height: 180 },
|
||||
};
|
||||
|
||||
export function videoSizeLabel(size: VideoSize): string {
|
||||
switch (size) {
|
||||
case "small":
|
||||
return "Small";
|
||||
case "medium":
|
||||
return "Medium";
|
||||
case "large":
|
||||
return "Large";
|
||||
case "full":
|
||||
return "Full screen";
|
||||
}
|
||||
}
|
||||
|
||||
/** Full screen needs the tall window band; the rest play in the standard one. */
|
||||
export function videoHeightMode(size: VideoSize): "min" | "max" {
|
||||
return size === "full" ? "max" : "min";
|
||||
}
|
||||
|
||||
/**
|
||||
* The largest size with the video's aspect ratio that fits in box, with even
|
||||
* dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel
|
||||
* columns) of at least 2. Upscales small videos too: the size is the
|
||||
* viewer's choice.
|
||||
*/
|
||||
export function fitVideo(video: Size, box: Size): Size {
|
||||
if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) };
|
||||
const scale = Math.min(box.width / video.width, box.height / video.height);
|
||||
return {
|
||||
width: evenFloor(Math.min(box.width, Math.round(video.width * scale))),
|
||||
height: evenFloor(Math.min(box.height, Math.round(video.height * scale))),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The video's rect within a viewport, centered. x snaps to a multiple of 4
|
||||
* and y to a multiple of 2 so frame deltas don't straddle an extra encoder
|
||||
* block at each edge.
|
||||
*/
|
||||
export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect {
|
||||
const fixed = FIXED_BOXES[size];
|
||||
const box = fixed
|
||||
? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) }
|
||||
: viewport;
|
||||
const fitted = fitVideo(video, box);
|
||||
return {
|
||||
x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4),
|
||||
y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2),
|
||||
...fitted,
|
||||
};
|
||||
}
|
||||
|
||||
/** "3:52", or "1:02:03" past an hour. */
|
||||
export function formatMediaTime(ms: number): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const seconds = String(total % 60).padStart(2, "0");
|
||||
return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`;
|
||||
}
|
||||
|
||||
function evenFloor(value: number): number {
|
||||
return Math.max(2, Math.floor(value / 2) * 2);
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { PHOTO_GAMMA } from "../../native/image-files";
|
||||
import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player";
|
||||
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
type InProcessAppOptions,
|
||||
type InProcessWindow,
|
||||
} from "../../ui/shell/in-process-window";
|
||||
import { appViewportSize } from "../../ui/shell/geometry";
|
||||
import {
|
||||
formatMediaTime,
|
||||
VIDEO_SIZES,
|
||||
videoHeightMode,
|
||||
videoRect,
|
||||
videoSizeLabel,
|
||||
type Rect,
|
||||
type VideoSize,
|
||||
} from "./video-layout";
|
||||
|
||||
export const videoSizeSetting = new ConfigSettingEnum<VideoSize>({
|
||||
id: "video-size",
|
||||
label: "Size",
|
||||
storageKey: "video.size",
|
||||
defaultValue: "large",
|
||||
values: VIDEO_SIZES,
|
||||
formatValue: videoSizeLabel,
|
||||
});
|
||||
|
||||
export const videoAudioSetting = new ConfigSettingBoolean({
|
||||
id: "video-audio",
|
||||
label: "Audio",
|
||||
storageKey: "video.audio",
|
||||
defaultValue: true,
|
||||
description: "Play the video's sound on the phone's current audio output.",
|
||||
});
|
||||
|
||||
export const videoMonochromeSetting = new ConfigSettingBoolean({
|
||||
id: "video-monochrome",
|
||||
label: "Monochrome",
|
||||
storageKey: "video.monochrome",
|
||||
defaultValue: false,
|
||||
description: "Show only black and white: far less to send per frame, so a higher frame rate.",
|
||||
});
|
||||
|
||||
/** How far one ring scroll seeks. */
|
||||
const SEEK_STEP_MS = 10_000;
|
||||
const MARGIN_X = 18;
|
||||
const TITLE_Y = 10;
|
||||
const PANEL_WIDTH = 340;
|
||||
|
||||
/**
|
||||
* One opened video: its probe result, the native player, and the window
|
||||
* glue. The window's base layer is the settings panel; playing pushes the
|
||||
* playback layer, and backing out of it (or the video ending) returns to the
|
||||
* panel with the position kept for Resume.
|
||||
*/
|
||||
class VideoSession {
|
||||
readonly probe: VideoInfo | { error: string };
|
||||
private player: VideoPlayer | null = null;
|
||||
private window: InProcessWindow | null = null;
|
||||
private playback: VideoPlaybackLayer | null = null;
|
||||
/** Where Resume starts; 0 = from the beginning. */
|
||||
resumeMs = 0;
|
||||
/** The last playback error, shown on the panel until the next Play. */
|
||||
error: string | null = null;
|
||||
|
||||
constructor(
|
||||
readonly path: string,
|
||||
readonly title: string,
|
||||
readonly surfaceId: string,
|
||||
) {
|
||||
this.probe = probeVideo(path);
|
||||
}
|
||||
|
||||
get info(): VideoInfo | null {
|
||||
return "error" in this.probe ? null : this.probe;
|
||||
}
|
||||
|
||||
attach(window: InProcessWindow): void {
|
||||
this.window = window;
|
||||
}
|
||||
|
||||
isPlaying(): boolean {
|
||||
return this.player?.isPlaying() ?? false;
|
||||
}
|
||||
|
||||
positionMs(): number {
|
||||
return this.player?.positionMs() ?? this.resumeMs;
|
||||
}
|
||||
|
||||
/** Current rect for a size choice, in the viewport of the band that size plays in. */
|
||||
rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect {
|
||||
const info = this.info;
|
||||
return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport);
|
||||
}
|
||||
|
||||
play(ctx: LayerContext, fromStart: boolean): void {
|
||||
const window = this.window;
|
||||
const info = this.info;
|
||||
if (!window || !info) return;
|
||||
this.error = null;
|
||||
const size = videoSizeSetting.get();
|
||||
window.setHeightMode(videoHeightMode(size));
|
||||
const viewport = ctx.stack.getBaseSize();
|
||||
const rect = this.rectFor(size, viewport);
|
||||
const player = this.ensurePlayer();
|
||||
this.playback = new VideoPlaybackLayer(this, rect, viewport);
|
||||
ctx.stack.push(this.playback);
|
||||
player.start({
|
||||
path: this.path,
|
||||
surfaceId: this.surfaceId,
|
||||
rect,
|
||||
monochrome: videoMonochromeSetting.get(),
|
||||
gamma: PHOTO_GAMMA,
|
||||
audio: info.hasAudio && videoAudioSetting.get(),
|
||||
startMs: fromStart ? 0 : this.resumeMs,
|
||||
});
|
||||
}
|
||||
|
||||
togglePause(): void {
|
||||
const player = this.player;
|
||||
if (!player) return;
|
||||
if (player.isPlaying()) player.pause();
|
||||
else player.resume();
|
||||
}
|
||||
|
||||
seekBy(deltaMs: number): void {
|
||||
const player = this.player;
|
||||
if (!player) return;
|
||||
const duration = this.info?.durationMs ?? 0;
|
||||
const target = Math.max(0, player.positionMs() + deltaMs);
|
||||
player.seekTo(duration > 0 ? Math.min(target, duration) : target);
|
||||
}
|
||||
|
||||
/** The window went to the background or the screen went off: stop, keep the place. */
|
||||
pauseForHost(): void {
|
||||
if (!this.player?.isPlaying()) return;
|
||||
this.player.pause();
|
||||
// Paint the paused overlay now, so the retained surface shows it on return.
|
||||
this.window?.requestRender();
|
||||
}
|
||||
|
||||
/** Leave the playback layer for the panel, remembering where to resume. */
|
||||
returnToPanel(resumeMs: number): void {
|
||||
const player = this.player;
|
||||
if (player) {
|
||||
player.pause();
|
||||
player.stop();
|
||||
}
|
||||
this.resumeMs = resumeMs;
|
||||
const window = this.window;
|
||||
const playback = this.playback;
|
||||
this.playback = null;
|
||||
if (window && playback) window.stack.popThrough(playback);
|
||||
window?.setHeightMode("min");
|
||||
window?.requestRender();
|
||||
}
|
||||
|
||||
lastFrame(rect: Rect): GrayImage | null {
|
||||
return this.player?.lastFrame(rect.width, rect.height) ?? null;
|
||||
}
|
||||
|
||||
release(): void {
|
||||
this.player?.release();
|
||||
this.player = null;
|
||||
}
|
||||
|
||||
private ensurePlayer(): VideoPlayer {
|
||||
if (!this.player) {
|
||||
this.player = new VideoPlayer({
|
||||
onState: (state, positionMs) => this.onPlayerState(state, positionMs),
|
||||
onError: (message) => {
|
||||
this.error = message;
|
||||
this.returnToPanel(this.positionMs());
|
||||
},
|
||||
});
|
||||
}
|
||||
return this.player;
|
||||
}
|
||||
|
||||
private onPlayerState(state: VideoPlaybackState, _positionMs: number): void {
|
||||
if (state === "ended") {
|
||||
this.returnToPanel(0);
|
||||
return;
|
||||
}
|
||||
// A paused seek decoded its new frame: repaint it under the overlay.
|
||||
if (state === "paused") this.window?.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean };
|
||||
|
||||
/**
|
||||
* The pre-play settings panel (the window's root): picture size, audio and
|
||||
* monochrome, all remembered for the next video, then Play (or Resume plus
|
||||
* Play from start once a position is kept).
|
||||
*/
|
||||
class VideoSettingsLayer implements Layer {
|
||||
private readonly menu = new Menu<PanelRow>({
|
||||
wrap: false,
|
||||
rowGap: 1,
|
||||
highlight: { radius: 6 },
|
||||
getHeight: () => listRowHeight(getDefaultSmallFont()),
|
||||
draw: (args) => this.drawRow(args),
|
||||
});
|
||||
constructor(private readonly session: VideoSession) {}
|
||||
|
||||
private rows(): PanelRow[] {
|
||||
if (!this.session.info) return [];
|
||||
const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }];
|
||||
rows.push({ kind: "play", fromStart: false });
|
||||
if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true });
|
||||
return rows;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220);
|
||||
const step = lineStep(font);
|
||||
let y = TITLE_Y + step;
|
||||
|
||||
const info = this.session.info;
|
||||
const errors: string[] = [];
|
||||
if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`);
|
||||
if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`);
|
||||
if (info) {
|
||||
const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"];
|
||||
image.drawText(font, MARGIN_X, y, facts.join(" · "), 130);
|
||||
y += step;
|
||||
}
|
||||
for (const error of errors) {
|
||||
for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) {
|
||||
image.drawText(font, MARGIN_X, y, line, 200);
|
||||
y += step;
|
||||
}
|
||||
}
|
||||
y += 6;
|
||||
|
||||
const rows = this.rows();
|
||||
const selected = this.menu.selectedItem;
|
||||
const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1;
|
||||
// Land on Resume/Play by default: the settings usually stay as they were.
|
||||
const playIndex = rows.findIndex((row) => row.kind === "play");
|
||||
this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined);
|
||||
const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6));
|
||||
this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) },
|
||||
ctx.stack.isFocused());
|
||||
return image;
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "scroll-down":
|
||||
await this.menu.handleInput(event);
|
||||
return;
|
||||
case "click": {
|
||||
const row = this.menu.selectedItem;
|
||||
if (row) this.activate(row, ctx);
|
||||
return;
|
||||
}
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private activate(row: PanelRow, ctx: LayerContext): void {
|
||||
switch (row.kind) {
|
||||
case "size":
|
||||
videoSizeSetting.set(videoSizeSetting.next());
|
||||
return;
|
||||
case "audio":
|
||||
if (this.session.info?.hasAudio) videoAudioSetting.toggle();
|
||||
return;
|
||||
case "monochrome":
|
||||
videoMonochromeSetting.toggle();
|
||||
return;
|
||||
case "play":
|
||||
this.session.play(ctx, row.fromStart);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs<PanelRow>): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const textX = x + 10;
|
||||
const textWidth = width - 20;
|
||||
switch (item.kind) {
|
||||
case "size": {
|
||||
const size = videoSizeSetting.get();
|
||||
// The picture size this choice gets in its own band (full screen plays in the tall window).
|
||||
const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size)));
|
||||
drawRightValueMenuItem(image, font, textX, y, textWidth, "Size",
|
||||
`${videoSizeLabel(size)} ${rect.width}x${rect.height}`);
|
||||
return;
|
||||
}
|
||||
case "audio": {
|
||||
const hasAudio = this.session.info?.hasAudio ?? false;
|
||||
drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)",
|
||||
hasAudio && videoAudioSetting.get(), selected);
|
||||
return;
|
||||
}
|
||||
case "monochrome":
|
||||
drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected);
|
||||
return;
|
||||
case "play": {
|
||||
const resumeMs = this.session.resumeMs;
|
||||
const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play";
|
||||
image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sameRow(a: PanelRow, b: PanelRow): boolean {
|
||||
return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart);
|
||||
}
|
||||
|
||||
/**
|
||||
* The playing video. The native player draws frames into the video rect on
|
||||
* its own; this layer paints only when the window renders (input, pause,
|
||||
* focus): black around the latest frame, plus a status box while paused.
|
||||
* Click pauses/resumes, scroll seeks 10s, double-click returns to the panel.
|
||||
*/
|
||||
class VideoPlaybackLayer implements Layer {
|
||||
private leaving = false;
|
||||
|
||||
constructor(
|
||||
private readonly session: VideoSession,
|
||||
private readonly rect: Rect,
|
||||
/** The viewport the rect was laid out in. */
|
||||
private readonly viewport: { width: number; height: number },
|
||||
) {}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) {
|
||||
// The window was relaid out (display mode change) and the native player's rect no
|
||||
// longer fits the surface: back out to the panel, which lays out afresh on Play.
|
||||
this.leaving = true;
|
||||
setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0);
|
||||
}
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const frame = this.session.lastFrame(this.rect);
|
||||
if (frame) image.bitBlt(frame, this.rect.x, this.rect.y);
|
||||
if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null);
|
||||
return image;
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent): Promise<void> {
|
||||
switch (event.type) {
|
||||
case "click":
|
||||
this.session.togglePause();
|
||||
return;
|
||||
case "scroll-up":
|
||||
this.session.seekBy(-SEEK_STEP_MS);
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.session.seekBy(SEEK_STEP_MS);
|
||||
return;
|
||||
case "double-click":
|
||||
this.session.returnToPanel(this.session.positionMs());
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "Paused 1:23 / 3:52" over a progress bar, below the picture when there is
|
||||
* room (small sizes), else over its bottom edge.
|
||||
*/
|
||||
private drawPausedBox(image: GrayImage, loading: boolean): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const duration = this.session.info?.durationMs ?? 0;
|
||||
const position = this.session.positionMs();
|
||||
const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`;
|
||||
const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24);
|
||||
const boxHeight = font.lineHeight + 14;
|
||||
const rect = this.rect;
|
||||
const x = Math.round(rect.x + (rect.width - boxWidth) / 2);
|
||||
const below = rect.y + rect.height + 4;
|
||||
const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6);
|
||||
image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6);
|
||||
image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6);
|
||||
image.drawText(font, x + 12, y + 5, text, 220);
|
||||
if (!loading && duration > 0) {
|
||||
const barWidth = boxWidth - 24;
|
||||
image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40);
|
||||
image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A video file's player as its own window, opened from the Files app. */
|
||||
export function createVideoPlayerWindow(
|
||||
windowId: string,
|
||||
title: string,
|
||||
path: string,
|
||||
options: InProcessAppOptions,
|
||||
): InProcessWindow {
|
||||
const session = new VideoSession(path, title, `window:${windowId}`);
|
||||
const window = createInProcessWindow({
|
||||
appId: "files",
|
||||
windowId,
|
||||
title,
|
||||
iconLetter: "V",
|
||||
icon: "film",
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)),
|
||||
// A playing video is the thing being watched: no idle screen-off.
|
||||
keepsScreenOn: () => session.isPlaying(),
|
||||
onForegroundChanged: (foreground) => {
|
||||
if (!foreground) session.pauseForHost();
|
||||
},
|
||||
setScreenOn: (on) => {
|
||||
if (!on) session.pauseForHost();
|
||||
},
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
reconfigureSurface: options.reconfigureSurface,
|
||||
onClosed: () => {
|
||||
session.release();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
session.attach(window);
|
||||
return window;
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Bridge to the Android FaceclawVideoPlayer: decodes a video file and
|
||||
* streams its frames, scaled to gray, straight into a rect of a window's
|
||||
* compositor surface, with the audio on the phone's default media route.
|
||||
* Android only; on iOS probeVideo reports an error and nothing plays.
|
||||
*/
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { GrayImage } from "../graphics/image";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
declare const global: any;
|
||||
|
||||
/** Containers MediaExtractor opens (AVI is not one of them). */
|
||||
const PLAYABLE_VIDEO = /\.(mp4|m4v|mkv|webm|3gp|mov)$/i;
|
||||
|
||||
export function isPlayableVideoFile(name: string): boolean {
|
||||
return PLAYABLE_VIDEO.test(name);
|
||||
}
|
||||
|
||||
export type VideoInfo = {
|
||||
/** Display size, rotation already applied. */
|
||||
width: number;
|
||||
height: number;
|
||||
durationMs: number;
|
||||
hasAudio: boolean;
|
||||
};
|
||||
|
||||
/** Read a video's size, duration and tracks; an error string when it can't be played. */
|
||||
export function probeVideo(path: string): VideoInfo | { error: string } {
|
||||
if (!global.isAndroid) return { error: "Video playback is not supported on this phone" };
|
||||
try {
|
||||
const raw = JSON.parse(String(com.faceclaw.app.FaceclawVideoPlayer.probe(path)));
|
||||
if (raw.error) return { error: String(raw.error) };
|
||||
return {
|
||||
width: Number(raw.width) || 0,
|
||||
height: Number(raw.height) || 0,
|
||||
durationMs: Number(raw.durationMs) || 0,
|
||||
hasAudio: Boolean(raw.hasAudio),
|
||||
};
|
||||
} catch (error) {
|
||||
return { error: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export type VideoPlaybackState = "playing" | "paused" | "ended";
|
||||
|
||||
export type VideoPlayerCallbacks = {
|
||||
/** State change, or a paused seek that landed on a new frame (state "paused"). */
|
||||
onState: (state: VideoPlaybackState, positionMs: number) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export type VideoStartOptions = {
|
||||
path: string;
|
||||
surfaceId: string;
|
||||
/** The video's rect in surface coordinates. */
|
||||
rect: { x: number; y: number; width: number; height: number };
|
||||
monochrome: boolean;
|
||||
/** Tone curve for gray output (ignored for monochrome); see PHOTO_GAMMA. */
|
||||
gamma: number;
|
||||
audio: boolean;
|
||||
startMs: number;
|
||||
};
|
||||
|
||||
/** One player; frames go to whatever surface the latest start() named. Main thread only. */
|
||||
export class VideoPlayer {
|
||||
private readonly player: any;
|
||||
// The Java listener proxy must stay referenced or it gets GC'd.
|
||||
private readonly listenerProxy: any;
|
||||
// Reused direct buffer (and its JS view) for copyLastFrame; see java-direct-buffer.ts
|
||||
// for why frames cross as a Java-allocated buffer.
|
||||
private frameBuffer: any = null;
|
||||
private frameView: Uint8Array | null = null;
|
||||
|
||||
constructor(callbacks: VideoPlayerCallbacks) {
|
||||
this.player = new com.faceclaw.app.FaceclawVideoPlayer(Utils.android.getApplicationContext());
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawVideoPlayer.Listener({
|
||||
onStateChanged: (state: string, positionMs: number) => {
|
||||
callbacks.onState(String(state) as VideoPlaybackState, Number(positionMs));
|
||||
},
|
||||
onError: (message: string) => callbacks.onError(String(message ?? "Playback failed")),
|
||||
});
|
||||
this.player.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
start(options: VideoStartOptions): void {
|
||||
const { rect } = options;
|
||||
this.player.start(
|
||||
options.path,
|
||||
options.surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
Math.round(rect.width),
|
||||
Math.round(rect.height),
|
||||
options.monochrome,
|
||||
options.gamma,
|
||||
options.audio,
|
||||
Math.max(0, Math.round(options.startMs)),
|
||||
);
|
||||
}
|
||||
|
||||
/** Once this returns, no more frames reach the surface until resume(). */
|
||||
pause(): void {
|
||||
this.player.pause();
|
||||
}
|
||||
|
||||
resume(): void {
|
||||
this.player.resume();
|
||||
}
|
||||
|
||||
seekTo(positionMs: number): void {
|
||||
this.player.seekTo(Math.max(0, Math.round(positionMs)));
|
||||
}
|
||||
|
||||
positionMs(): number {
|
||||
return Number(this.player.getPositionMs());
|
||||
}
|
||||
|
||||
isPlaying(): boolean {
|
||||
return Boolean(this.player.isPlaying());
|
||||
}
|
||||
|
||||
/** The latest decoded frame at the given size, or null before one exists. */
|
||||
lastFrame(width: number, height: number): GrayImage | null {
|
||||
const bytes = width * height;
|
||||
if (bytes <= 0) return null;
|
||||
if (!this.frameView || this.frameView.length !== bytes) {
|
||||
this.frameBuffer = java.nio.ByteBuffer.allocateDirect(bytes);
|
||||
this.frameView = new Uint8Array((ArrayBuffer as any).from(this.frameBuffer));
|
||||
}
|
||||
if (!this.player.copyLastFrame(this.frameBuffer)) return null;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
image.pixels.set(this.frameView);
|
||||
return image;
|
||||
}
|
||||
|
||||
/** End playback and free the decoder and audio (the last frame stays readable). */
|
||||
stop(): void {
|
||||
this.player.stop();
|
||||
}
|
||||
|
||||
release(): void {
|
||||
this.player.release();
|
||||
this.frameBuffer = null;
|
||||
this.frameView = null;
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,10 @@ export type InProcessWindowOptions = {
|
||||
onSystemMenuOpened?: () => void;
|
||||
onAppMenuOpened?: () => void;
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** See ShellWindow.keepsScreenOn. */
|
||||
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||
onForegroundChanged?: (foreground: boolean) => void;
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
actions: LayerActions;
|
||||
/**
|
||||
@@ -162,6 +166,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
acceptsDirectional: true,
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
keepsScreenOn: options.keepsScreenOn,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
@@ -218,6 +223,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
onFocus: options.onFocus,
|
||||
setForeground: (foreground) => {
|
||||
options.setSurfaceVisible(foreground);
|
||||
options.onForegroundChanged?.(foreground);
|
||||
},
|
||||
};
|
||||
const setHeightMode = (mode: WindowHeightMode) => {
|
||||
|
||||
+10
-1
@@ -101,6 +101,12 @@ export type ShellWindow = {
|
||||
holdToTalk?: boolean;
|
||||
/** A window owns microphone capture outside the shell voice dialog. */
|
||||
isVoiceCapturing?: () => boolean;
|
||||
/**
|
||||
* True while the window shows something the wearer watches without
|
||||
* touching anything (a playing video): the idle screen timeout waits, as
|
||||
* it does for voice capture, while this window is in the foreground.
|
||||
*/
|
||||
keepsScreenOn?: () => boolean;
|
||||
/**
|
||||
* True when the window gives swipe-left / swipe-right (watch directional
|
||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||
@@ -647,7 +653,10 @@ class Shell {
|
||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||
// tool loop can run for a while with no input); once the turn ends and
|
||||
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
||||
// A foreground window that is playing video holds it off the same way.
|
||||
const foreground = this.foregroundWindow();
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
|
||||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* Scales one decoded video frame's luma plane into an 8bpp gray rect for a compositor surface:
|
||||
* a box filter (averaging up to [MAX_TAPS] x [MAX_TAPS] evenly spaced samples per output pixel,
|
||||
* nearest-neighbor when upscaling), then the container's display rotation, then a tone lookup.
|
||||
*
|
||||
* Only luma is read: the glasses show gray, and Y is exactly the gray a decoder hands back
|
||||
* without any color conversion. Platform code owns decoding and copies the plane into a
|
||||
* ByteArray; this class owns the pixel math. One instance serves one (source size, rotation,
|
||||
* output size) combination and reuses its scratch buffers across frames; not thread-safe.
|
||||
*/
|
||||
class VideoFrameScaler(
|
||||
/** Visible luma size in coded (unrotated) orientation. */
|
||||
val sourceWidth: Int,
|
||||
val sourceHeight: Int,
|
||||
/** Clockwise display rotation in degrees: 0, 90, 180 or 270. */
|
||||
val rotation: Int,
|
||||
/** Output size in display orientation. */
|
||||
val outputWidth: Int,
|
||||
val outputHeight: Int,
|
||||
) {
|
||||
companion object {
|
||||
/** Most samples averaged along each axis per output pixel; bounds work for big downscales. */
|
||||
const val MAX_TAPS = 4
|
||||
|
||||
/** Snap any angle to the nearest of 0/90/180/270, the only rotations containers use. */
|
||||
@JvmStatic
|
||||
fun normalizeRotation(degrees: Int): Int = (((((degrees % 360) + 360) % 360 + 45) / 90) % 4) * 90
|
||||
|
||||
/**
|
||||
* Luma to display gray. Limited ("TV") range video codes black at 16 and white at 235, so
|
||||
* it is stretched to 0..255 first. Monochrome thresholds at mid-gray to 0 or 255 (a
|
||||
* 1-bit image compresses and delta-encodes far better than 16 levels); otherwise the
|
||||
* gamma curve applies, as for photos (see GrayPacket.fromArgb).
|
||||
*/
|
||||
@JvmStatic
|
||||
fun toneLut(limitedRange: Boolean, gamma: Float, monochrome: Boolean): ByteArray {
|
||||
val curve = GrayPacket.toneCurve(gamma)
|
||||
val lut = ByteArray(256)
|
||||
for (y in 0 until 256) {
|
||||
val full = if (limitedRange) ((y - 16) * 255 + 109) / 219 else y
|
||||
val v = full.coerceIn(0, 255)
|
||||
lut[y] = (if (monochrome) (if (v >= 128) 255 else 0) else curve[v]).toByte()
|
||||
}
|
||||
return lut
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample positions for each output index along one axis: [count] entries per output,
|
||||
* evenly spaced inside that output's box of source pixels.
|
||||
*/
|
||||
private fun taps(source: Int, output: Int, count: Int): IntArray {
|
||||
val positions = IntArray(output * count)
|
||||
for (i in 0 until output) {
|
||||
for (j in 0 until count) {
|
||||
// Center of sub-box j of output i, in source pixels: (i + (j + 0.5)/count) * source / output.
|
||||
val numerator = (i.toLong() * count + j) * 2 + 1
|
||||
val position = (numerator * source / (2L * count * output)).toInt()
|
||||
positions[i * count + j] = position.coerceIn(0, source - 1)
|
||||
}
|
||||
}
|
||||
return positions
|
||||
}
|
||||
}
|
||||
|
||||
private val swapsAxes = rotation == 90 || rotation == 270
|
||||
/** Scaled size before rotation (coded orientation). */
|
||||
private val scaledWidth = if (swapsAxes) outputHeight else outputWidth
|
||||
private val scaledHeight = if (swapsAxes) outputWidth else outputHeight
|
||||
private val tapsX = ((sourceWidth + scaledWidth - 1) / scaledWidth).coerceIn(1, MAX_TAPS)
|
||||
private val tapsY = ((sourceHeight + scaledHeight - 1) / scaledHeight).coerceIn(1, MAX_TAPS)
|
||||
private val xs = taps(sourceWidth, scaledWidth, tapsX)
|
||||
private val ys = taps(sourceHeight, scaledHeight, tapsY)
|
||||
private val sums = IntArray(scaledWidth)
|
||||
/** Unrotated scaled frame, needed only when a rotation follows. */
|
||||
private val scaled: ByteArray? = if (rotation == 0) null else ByteArray(scaledWidth * scaledHeight)
|
||||
|
||||
init {
|
||||
require(sourceWidth > 0 && sourceHeight > 0) { "bad source size ${sourceWidth}x$sourceHeight" }
|
||||
require(outputWidth > 0 && outputHeight > 0) { "bad output size ${outputWidth}x$outputHeight" }
|
||||
require(rotation == 0 || rotation == 90 || rotation == 180 || rotation == 270) { "bad rotation $rotation" }
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale the luma plane into [out] (outputWidth * outputHeight bytes, row-major). The visible
|
||||
* pixel (x, y) is at luma[offset + y * rowStride + x * pixelStride]; [lut] is a 256-entry
|
||||
* table from [toneLut].
|
||||
*/
|
||||
fun scale(luma: ByteArray, offset: Int, rowStride: Int, pixelStride: Int, lut: ByteArray, out: ByteArray) {
|
||||
require(out.size >= outputWidth * outputHeight) { "output buffer too small" }
|
||||
val target = scaled ?: out
|
||||
val count = tapsX * tapsY
|
||||
val half = count / 2
|
||||
for (oy in 0 until scaledHeight) {
|
||||
sums.fill(0)
|
||||
for (ty in 0 until tapsY) {
|
||||
val row = offset + ys[oy * tapsY + ty] * rowStride
|
||||
var tap = 0
|
||||
for (ox in 0 until scaledWidth) {
|
||||
var sum = 0
|
||||
for (tx in 0 until tapsX) {
|
||||
sum += luma[row + xs[tap++] * pixelStride].toInt() and 0xff
|
||||
}
|
||||
sums[ox] += sum
|
||||
}
|
||||
}
|
||||
val base = oy * scaledWidth
|
||||
for (ox in 0 until scaledWidth) {
|
||||
target[base + ox] = lut[(sums[ox] + half) / count]
|
||||
}
|
||||
}
|
||||
if (scaled != null) rotate(scaled, out)
|
||||
}
|
||||
|
||||
/** Rotate the coded-orientation frame clockwise by [rotation] into display orientation. */
|
||||
private fun rotate(source: ByteArray, out: ByteArray) {
|
||||
val w = scaledWidth
|
||||
val h = scaledHeight
|
||||
for (y in 0 until outputHeight) {
|
||||
val base = y * outputWidth
|
||||
for (x in 0 until outputWidth) {
|
||||
out[base + x] = when (rotation) {
|
||||
90 -> source[(h - 1 - x) * w + y]
|
||||
180 -> source[(h - 1 - y) * w + (w - 1 - x)]
|
||||
else -> source[x * w + (w - 1 - y)]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
|
||||
const load = loader({ global: { isIOS: true } }, {
|
||||
'../../native/file-access': {}, '../../native/font-files': { isFontFile: () => false },
|
||||
'../../graphics/installed-fonts': {}, '../../native/image-files': { isDecodableImageFile: () => false },
|
||||
'../../native/video-player': { isPlayableVideoFile: () => true },
|
||||
'../evenhub/installed-apps': { readEvenHubPackageManifest: () => null }, '../evenhub/permission-dialog': {},
|
||||
'./file-browser': { FileBrowserLayer: class { constructor(options) { browserOptions = options; } } },
|
||||
'./file-info-dialog': { FileInfoDialogLayer: class { constructor(entry, value) { actions = value; } } },
|
||||
@@ -64,6 +65,8 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path'
|
||||
});
|
||||
load('app/apps/files/files-app.ts').createFilesAppWindow({ openEhpkApp: path => { opened = path; }, installEhpkApp: path => { installed = path; } });
|
||||
assert.equal(browserOptions.isSupportedFile('test.EHPK'), true);
|
||||
// Video playback is Android-only for now.
|
||||
assert.equal(browserOptions.isSupportedFile('clip.mp4'), false);
|
||||
const ctx = { stack: { push() {}, pop() {} } };
|
||||
browserOptions.onFilePicked({ name: 'test.ehpk', path: '/Documents/test.ehpk' }, ctx);
|
||||
assert.deepEqual(Array.from(actions, a => a.label), ['Run app', 'Install']);
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.*
|
||||
|
||||
class VideoFrameScalerTest {
|
||||
private val identity = ByteArray(256) { it.toByte() }
|
||||
|
||||
private fun scale(scaler: VideoFrameScaler, luma: ByteArray, lut: ByteArray = identity): IntArray {
|
||||
val out = ByteArray(scaler.outputWidth * scaler.outputHeight)
|
||||
scaler.scale(luma, 0, scaler.sourceWidth, 1, lut, out)
|
||||
return IntArray(out.size) { out[it].toInt() and 0xff }
|
||||
}
|
||||
|
||||
@Test fun twoToOneDownscaleAveragesEachBox() {
|
||||
// 4x2 source -> 2x1: each output is the mean of a 2x2 box.
|
||||
val luma = byteArrayOf(0, 100, 200.toByte(), 200.toByte(), 20, 80, 100, 100)
|
||||
val scaler = VideoFrameScaler(4, 2, 0, 2, 1)
|
||||
assertContentEquals(intArrayOf(50, 150), scale(scaler, luma))
|
||||
}
|
||||
|
||||
@Test fun largeDownscaleStaysWithinTheTapBudget() {
|
||||
// 100:1 horizontally still averages only MAX_TAPS samples, spread across the box.
|
||||
val luma = ByteArray(200) { (if (it % 100 < 50) 0 else 200).toByte() }
|
||||
val scaler = VideoFrameScaler(200, 1, 0, 2, 1)
|
||||
assertContentEquals(intArrayOf(100, 100), scale(scaler, luma))
|
||||
}
|
||||
|
||||
@Test fun upscaleIsNearestNeighbor() {
|
||||
val scaler = VideoFrameScaler(2, 1, 0, 4, 2)
|
||||
assertContentEquals(intArrayOf(10, 10, 90, 90, 10, 10, 90, 90), scale(scaler, byteArrayOf(10, 90)))
|
||||
}
|
||||
|
||||
@Test fun offsetAndStridesSelectTheVisiblePlane() {
|
||||
// A 2x2 visible crop at (1, 1) of a 4-wide plane, with interleaved samples (pixelStride 2).
|
||||
val plane = ByteArray(40)
|
||||
fun put(x: Int, y: Int, v: Int) { plane[(1 + y) * 10 + 1 + x * 2] = v.toByte() }
|
||||
put(0, 0, 11); put(1, 0, 22); put(0, 1, 33); put(1, 1, 44)
|
||||
val scaler = VideoFrameScaler(2, 2, 0, 2, 2)
|
||||
val out = ByteArray(4)
|
||||
scaler.scale(plane, 11, 10, 2, identity, out)
|
||||
assertContentEquals(byteArrayOf(11, 22, 33, 44), out)
|
||||
}
|
||||
|
||||
@Test fun rotationsMatchTheContainerClockwiseConvention() {
|
||||
// Coded 3x2: 1 2 3
|
||||
// 4 5 6
|
||||
val luma = byteArrayOf(1, 2, 3, 4, 5, 6)
|
||||
assertContentEquals(intArrayOf(4, 1, 5, 2, 6, 3), scale(VideoFrameScaler(3, 2, 90, 2, 3), luma))
|
||||
assertContentEquals(intArrayOf(6, 5, 4, 3, 2, 1), scale(VideoFrameScaler(3, 2, 180, 3, 2), luma))
|
||||
assertContentEquals(intArrayOf(3, 6, 2, 5, 1, 4), scale(VideoFrameScaler(3, 2, 270, 2, 3), luma))
|
||||
}
|
||||
|
||||
@Test fun normalizeRotationSnapsToQuarterTurns() {
|
||||
assertEquals(0, VideoFrameScaler.normalizeRotation(0))
|
||||
assertEquals(90, VideoFrameScaler.normalizeRotation(90))
|
||||
assertEquals(270, VideoFrameScaler.normalizeRotation(-90))
|
||||
assertEquals(180, VideoFrameScaler.normalizeRotation(540))
|
||||
assertEquals(0, VideoFrameScaler.normalizeRotation(359))
|
||||
}
|
||||
|
||||
@Test fun limitedRangeStretchesAndMonochromeThresholds() {
|
||||
val gray = VideoFrameScaler.toneLut(limitedRange = true, gamma = 1f, monochrome = false)
|
||||
assertEquals(0, gray[16].toInt() and 0xff)
|
||||
assertEquals(0, gray[0].toInt() and 0xff)
|
||||
assertEquals(255, gray[235].toInt() and 0xff)
|
||||
assertEquals(255, gray[255].toInt() and 0xff)
|
||||
val mono = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = true)
|
||||
assertEquals(0, mono[127].toInt() and 0xff)
|
||||
assertEquals(255, mono[128].toInt() and 0xff)
|
||||
// Gamma darkens midtones but leaves the ends alone.
|
||||
val photo = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = false)
|
||||
assertTrue((photo[128].toInt() and 0xff) < 64)
|
||||
assertEquals(255, photo[255].toInt() and 0xff)
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@
|
||||
"../app/g2/glasses-display-state.ts",
|
||||
"../app/g2/glance-state.ts",
|
||||
"../app/apps/glanceboard/layout.ts",
|
||||
"../app/apps/files/video-layout.ts",
|
||||
"../app/g2/pairing-candidates.ts",
|
||||
"../app/apps/calculator/math/*.ts",
|
||||
"../app/apps/microphones/mic-protocol.ts",
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
const {
|
||||
fitVideo,
|
||||
formatMediaTime,
|
||||
videoHeightMode,
|
||||
videoRect,
|
||||
} = require("../.test-build/app/apps/files/video-layout.js");
|
||||
|
||||
const WINDOW = { width: 576, height: 260 };
|
||||
const TALL_WINDOW = { width: 576, height: 452 };
|
||||
// ../g2flash/demos/badapple.mp4
|
||||
const BAD_APPLE = { width: 1444, height: 1080 };
|
||||
|
||||
test("fixed sizes fit their box and keep the aspect ratio", () => {
|
||||
assert.deepEqual(videoRect("small", BAD_APPLE, WINDOW), { x: 208, y: 70, width: 160, height: 120 });
|
||||
assert.deepEqual(videoRect("medium", BAD_APPLE, WINDOW), { x: 168, y: 40, width: 240, height: 180 });
|
||||
const wide = videoRect("small", { width: 1920, height: 1080 }, WINDOW);
|
||||
assert.equal(wide.width, 160);
|
||||
assert.equal(wide.height, 90);
|
||||
});
|
||||
|
||||
test("large fills the window height; full screen fills the tall window's width", () => {
|
||||
assert.deepEqual(videoRect("large", BAD_APPLE, WINDOW), { x: 112, y: 0, width: 348, height: 260 });
|
||||
assert.deepEqual(videoRect("full", BAD_APPLE, TALL_WINDOW), { x: 0, y: 10, width: 576, height: 430 });
|
||||
assert.equal(videoHeightMode("full"), "max");
|
||||
assert.equal(videoHeightMode("large"), "min");
|
||||
});
|
||||
|
||||
test("rects are aligned for the delta encoder and stay inside the viewport", () => {
|
||||
for (const size of ["small", "medium", "large", "full"]) {
|
||||
for (const video of [BAD_APPLE, { width: 1080, height: 1920 }, { width: 100, height: 20 }, { width: 7, height: 5 }]) {
|
||||
for (const viewport of [WINDOW, TALL_WINDOW, { width: 150, height: 100 }]) {
|
||||
const rect = videoRect(size, video, viewport);
|
||||
const where = `${size} ${video.width}x${video.height} in ${viewport.width}x${viewport.height}`;
|
||||
assert.equal(rect.x % 4, 0, where);
|
||||
assert.equal(rect.y % 2, 0, where);
|
||||
assert.equal(rect.width % 2, 0, where);
|
||||
assert.equal(rect.height % 2, 0, where);
|
||||
assert.ok(rect.width >= 2 && rect.height >= 2, where);
|
||||
assert.ok(rect.x + rect.width <= viewport.width, where);
|
||||
assert.ok(rect.y + rect.height <= viewport.height, where);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("a tiny video is scaled up to the chosen size", () => {
|
||||
assert.deepEqual(fitVideo({ width: 80, height: 60 }, { width: 240, height: 180 }), { width: 240, height: 180 });
|
||||
});
|
||||
|
||||
test("media times", () => {
|
||||
assert.equal(formatMediaTime(0), "0:00");
|
||||
assert.equal(formatMediaTime(232_130), "3:52");
|
||||
assert.equal(formatMediaTime(3_723_000), "1:02:03");
|
||||
});
|
||||
Reference in new issue
Block a user