diff --git a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java index eb9c8bb08b4..16497da7d9d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java @@ -53,4 +53,22 @@ PreviewingVideoGraph create( long initialTimestampOffsetUs) throws VideoFrameProcessingException; } + + /** + * Renders the oldest unrendered output frame that has become {@linkplain + * Listener#onOutputFrameAvailableForRendering(long) available for rendering} at the given {@code + * renderTimeNs}. + * + *

This will either render the output frame to the {@linkplain #setOutputSurfaceInfo output + * surface}, or drop the frame, per {@code renderTimeNs}. + * + *

The {@code renderTimeNs} may be passed to {@link + * android.opengl.EGLExt#eglPresentationTimeANDROID} depending on the implementation. + * + * @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can + * be before or after the current system time. Use {@link + * VideoFrameProcessor#DROP_OUTPUT_FRAME} to drop the frame, or {@link + * VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately. + */ + void renderOutputFrame(long renderTimeNs); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java index f0568edfe77..fcaa96e8945 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java @@ -98,4 +98,9 @@ private PreviewingSingleInputVideoGraph( presentation, initialTimestampOffsetUs); } + + @Override + public void renderOutputFrame(long renderTimeNs) { + getProcessor(SINGLE_INPUT_INDEX).renderOutputFrame(renderTimeNs); + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 15fa3f1e116..bcf36638e09 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -31,6 +31,7 @@ import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.common.util.UnstableApi; +import com.google.common.util.concurrent.MoreExecutors; import java.util.List; import java.util.concurrent.Executor; @@ -54,6 +55,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph { @Nullable private final Presentation presentation; @Nullable private VideoFrameProcessor videoFrameProcessor; + private boolean isEnded; private boolean released; private volatile boolean hasProducedFrameWithTimestampZero; @@ -112,7 +114,7 @@ public int registerInput() throws VideoFrameProcessingException { inputColorInfo, outputColorInfo, renderFramesAutomatically, - listenerExecutor, + /* listenerExecutor= */ MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { private long lastProcessedFramePresentationTimeUs; @@ -129,6 +131,12 @@ public void onOutputSizeChanged(int width, int height) { @Override public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (isEnded) { + onError( + new VideoFrameProcessingException( + "onOutputFrameAvailableForRendering() received after onEnded()")); + return; + } // Frames are rendered automatically. if (presentationTimeUs == 0) { hasProducedFrameWithTimestampZero = true; @@ -145,7 +153,13 @@ public void onError(VideoFrameProcessingException exception) { @Override public void onEnded() { - listener.onEnded(lastProcessedFramePresentationTimeUs); + if (isEnded) { + onError(new VideoFrameProcessingException("onEnded() received multiple times")); + return; + } + isEnded = true; + listenerExecutor.execute( + () -> listener.onEnded(lastProcessedFramePresentationTimeUs)); } }); return SINGLE_INPUT_INDEX; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java index 120535476d0..31e83cd4788 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -15,16 +15,16 @@ */ package androidx.media3.exoplayer.video; -import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.graphics.Bitmap; -import android.os.Handler; +import android.os.Looper; import android.util.Pair; import android.view.Surface; +import androidx.annotation.FloatRange; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -41,9 +41,8 @@ import androidx.media3.common.VideoGraph; import androidx.media3.common.VideoSize; import androidx.media3.common.util.Clock; -import androidx.media3.common.util.LongArrayQueue; +import androidx.media3.common.util.HandlerWrapper; import androidx.media3.common.util.Size; -import androidx.media3.common.util.TimedValueQueue; import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -53,6 +52,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.concurrent.Executor; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; @@ -60,15 +60,27 @@ /** Handles composition of video sinks. */ @UnstableApi -/* package */ final class CompositingVideoSinkProvider implements VideoSinkProvider { +/* package */ final class CompositingVideoSinkProvider + implements VideoSinkProvider, VideoGraph.Listener, VideoFrameRenderControl.FrameRenderer { + + private static final Executor NO_OP_EXECUTOR = runnable -> {}; private final Context context; private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; private final VideoFrameReleaseControl videoFrameReleaseControl; - - @Nullable private VideoSinkImpl videoSinkImpl; - @Nullable private List videoEffects; - @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; + private final VideoFrameRenderControl videoFrameRenderControl; + + private Clock clock; + private @MonotonicNonNull Format outputFormat; + private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener; + private @MonotonicNonNull HandlerWrapper handler; + private @MonotonicNonNull PreviewingVideoGraph videoGraph; + private @MonotonicNonNull VideoSinkImpl videoSinkImpl; + private @MonotonicNonNull List videoEffects; + @Nullable private Pair currentSurfaceAndSize; + private VideoSink.Listener listener; + private Executor listenerExecutor; + private int pendingFlushCount; private boolean released; /** Creates a new instance. */ @@ -90,24 +102,54 @@ public CompositingVideoSinkProvider( this.context = context; this.previewingVideoGraphFactory = previewingVideoGraphFactory; this.videoFrameReleaseControl = videoFrameReleaseControl; + @SuppressWarnings("nullness:assignment") + VideoFrameRenderControl.@Initialized FrameRenderer thisRef = this; + videoFrameRenderControl = + new VideoFrameRenderControl(/* frameRenderer= */ thisRef, videoFrameReleaseControl); + clock = Clock.DEFAULT; + listener = VideoSink.Listener.NO_OP; + listenerExecutor = NO_OP_EXECUTOR; } + // VideoSinkProvider methods + @Override public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException { checkState(!released && videoSinkImpl == null); checkStateNotNull(videoEffects); + // Lazily initialize the handler here so it's initialized on the playback looper. + handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null); + + ColorInfo inputColorInfo = + sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo) + ? sourceFormat.colorInfo + : ColorInfo.SDR_BT709_LIMITED; + ColorInfo outputColorInfo = inputColorInfo; + if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { + // SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ. + outputColorInfo = + inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build(); + } try { + @SuppressWarnings("nullness:assignment") + VideoGraph.@Initialized Listener thisRef = this; + videoGraph = + previewingVideoGraphFactory.create( + context, + inputColorInfo, + outputColorInfo, + DebugViewProvider.NONE, + /* listener= */ thisRef, + /* listenerExecutor= */ handler::post, + /* compositionEffects= */ ImmutableList.of(), + /* initialTimestampOffsetUs= */ 0); videoSinkImpl = new VideoSinkImpl( - context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat); + context, /* compositingVideoSinkProvider= */ this, videoGraph, sourceFormat); } catch (VideoFrameProcessingException e) { throw new VideoSink.VideoSinkException(e, sourceFormat); } - - if (videoFrameMetadataListener != null) { - videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener); - } videoSinkImpl.setVideoEffects(checkNotNull(videoEffects)); } @@ -122,10 +164,16 @@ public void release() { return; } + if (handler != null) { + handler.removeCallbacksAndMessages(/* token= */ null); + } if (videoSinkImpl != null) { videoSinkImpl.release(); - videoSinkImpl = null; } + if (videoGraph != null) { + videoGraph.release(); + } + currentSurfaceAndSize = null; released = true; } @@ -157,138 +205,222 @@ public void setStreamOffsetUs(long streamOffsetUs) { @Override public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { - checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution); + if (currentSurfaceAndSize != null + && currentSurfaceAndSize.first.equals(outputSurface) + && currentSurfaceAndSize.second.equals(outputResolution)) { + return; + } + videoFrameReleaseControl.setOutputSurface(outputSurface); + currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); + checkStateNotNull(videoGraph) + .setOutputSurfaceInfo( + new SurfaceInfo( + outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); } @Override public void clearOutputSurfaceInfo() { - checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo(); + checkStateNotNull(videoGraph).setOutputSurfaceInfo(/* outputSurfaceInfo= */ null); + currentSurfaceAndSize = null; } @Override public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) { this.videoFrameMetadataListener = videoFrameMetadataListener; - if (isInitialized()) { - checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener); + } + + @Override + public void setClock(Clock clock) { + checkState(!isInitialized()); + this.clock = clock; + } + + // VideoGraph.Listener + + @Override + public void onOutputSizeChanged(int width, int height) { + // We forward output size changes to render control even if we are still flushing. + videoFrameRenderControl.onOutputSizeChanged(width, height); + } + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (pendingFlushCount > 0) { + // Ignore available frames while the sink provider is flushing + return; + } + videoFrameRenderControl.onOutputFrameAvailableForRendering(presentationTimeUs); + } + + @Override + public void onEnded(long finalFramePresentationTimeUs) { + throw new UnsupportedOperationException(); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + VideoSink.Listener currentListener = this.listener; + listenerExecutor.execute( + () -> { + VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl); + currentListener.onError( + videoSink, + new VideoSink.VideoSinkException( + exception, checkStateNotNull(videoSink.inputFormat))); + }); + } + + // FrameRenderer methods + + @Override + public void onVideoSizeChanged(VideoSize videoSize) { + outputFormat = + new Format.Builder() + .setWidth(videoSize.width) + .setHeight(videoSize.height) + .setSampleMimeType(MimeTypes.VIDEO_RAW) + .build(); + VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl); + VideoSink.Listener currentListener = this.listener; + listenerExecutor.execute(() -> currentListener.onVideoSizeChanged(videoSink, videoSize)); + } + + @Override + public void renderFrame( + long renderTimeNs, long bufferPresentationTimeUs, long streamOffsetUs, boolean isFirstFrame) { + if (isFirstFrame && listenerExecutor != NO_OP_EXECUTOR) { + VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl); + VideoSink.Listener currentListener = this.listener; + listenerExecutor.execute(() -> currentListener.onFirstFrameRendered(videoSink)); } + if (videoFrameMetadataListener != null) { + // TODO b/292111083 - outputFormat is initialized after the first frame is rendered because + // onVideoSizeChanged is announced after the first frame is available for rendering. + Format format = outputFormat == null ? new Format.Builder().build() : outputFormat; + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + /* presentationTimeUs= */ bufferPresentationTimeUs - streamOffsetUs, + clock.nanoTime(), + format, + /* mediaFormat= */ null); + } + checkStateNotNull(videoGraph).renderOutputFrame(renderTimeNs); } - private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener { + @Override + public void dropFrame() { + VideoSink.Listener currentListener = this.listener; + listenerExecutor.execute( + () -> currentListener.onFrameDropped(checkStateNotNull(videoSinkImpl))); + checkStateNotNull(videoGraph).renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME); + } + + // Internal methods + + private void setListener(VideoSink.Listener listener, Executor executor) { + if (Objects.equals(listener, this.listener)) { + checkState(Objects.equals(executor, listenerExecutor)); + return; + } + this.listener = listener; + this.listenerExecutor = executor; + } + + private boolean isReady() { + return pendingFlushCount == 0 && videoFrameRenderControl.isReady(); + } + + private boolean hasReleasedFrame(long presentationTimeUs) { + return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs); + } + + private void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + if (pendingFlushCount == 0) { + videoFrameRenderControl.render(positionUs, elapsedRealtimeUs); + } + } + + private void flush() { + pendingFlushCount++; + // Flush the render control now to ensure it has no data, eg calling isReady() must return false + // and + // render() should not render any frames. + videoFrameRenderControl.flush(); + // Finish flushing after handling pending video graph callbacks to ensure video size changes + // reach the video render control. + checkStateNotNull(handler).post(this::flushInternal); + } + + private void flushInternal() { + pendingFlushCount--; + if (pendingFlushCount > 0) { + // Another flush has been issued. + return; + } else if (pendingFlushCount < 0) { + throw new IllegalStateException(String.valueOf(pendingFlushCount)); + } + // Flush the render control again. + videoFrameRenderControl.flush(); + } + + private void setPlaybackSpeed(float speed) { + videoFrameRenderControl.setPlaybackSpeed(speed); + } + + private void onStreamOffsetChange(long bufferPresentationTimeUs, long streamOffsetUs) { + videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs); + } + + /** Receives input from an ExoPlayer renderer and forwards it to the video graph. */ + private static final class VideoSinkImpl implements VideoSink { private final Context context; - private final VideoFrameReleaseControl videoFrameReleaseControl; - private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; + private final CompositingVideoSinkProvider compositingVideoSinkProvider; private final VideoFrameProcessor videoFrameProcessor; - private final LongArrayQueue processedFramesBufferTimestampsUs; - private final TimedValueQueue streamOffsets; - private final TimedValueQueue videoSizeChanges; - private final Handler handler; private final int videoFrameProcessorMaxPendingFrameCount; private final ArrayList videoEffects; @Nullable private final Effect rotationEffect; - private VideoSink.@MonotonicNonNull Listener listener; - private @MonotonicNonNull Executor listenerExecutor; - @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; @Nullable private Format inputFormat; - @Nullable private Pair currentSurfaceAndSize; + private long inputStreamOffsetUs; + private boolean pendingInputStreamOffsetChange; - /** - * Whether the last frame of the current stream is decoded and registered to {@link - * VideoFrameProcessor}. - */ - private boolean registeredLastFrame; + /** The buffer presentation time, in microseconds, of the final frame in the stream. */ + private long finalBufferPresentationTimeUs; /** - * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}. + * The buffer presentation timestamp, in microseconds, of the most recently registered frame. */ - private boolean processedLastFrame; + private long lastBufferPresentationTimeUs; - /** Whether the last frame of the current stream is released to the output {@link Surface}. */ - private boolean releasedLastFrame; - - private long lastCodecBufferPresentationTimestampUs; - private VideoSize processedFrameSize; - private VideoSize reportedVideoSize; - private boolean pendingVideoSizeChange; - private long inputStreamOffsetUs; - private boolean pendingInputStreamOffsetChange; - private long outputStreamOffsetUs; - - // TODO b/292111083 - Remove the field and trigger the callback on every video size change. - private boolean onVideoSizeChangedCalled; private boolean hasRegisteredFirstInputStream; - private boolean inputStreamRegistrationPending; - private long lastFramePresentationTimeUs; + private long pendingInputStreamBufferPresentationTimeUs; /** Creates a new instance. */ public VideoSinkImpl( Context context, - PreviewingVideoGraph.Factory previewingVideoGraphFactory, - VideoFrameReleaseControl videoFrameReleaseControl, + CompositingVideoSinkProvider compositingVideoSinkProvider, + PreviewingVideoGraph videoGraph, Format sourceFormat) throws VideoFrameProcessingException { this.context = context; - this.videoFrameReleaseControl = videoFrameReleaseControl; - videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); - processedFramesBufferTimestampsUs = new LongArrayQueue(); - streamOffsets = new TimedValueQueue<>(); - videoSizeChanges = new TimedValueQueue<>(); + this.compositingVideoSinkProvider = compositingVideoSinkProvider; // TODO b/226330223 - Investigate increasing frame count when frame dropping is // allowed. // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed // reduces decoder timeouts, and consider restoring. videoFrameProcessorMaxPendingFrameCount = Util.getMaxPendingFramesCountForMediaCodecDecoders(context); - lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; - processedFrameSize = VideoSize.UNKNOWN; - reportedVideoSize = VideoSize.UNKNOWN; - - // Playback thread handler. - handler = Util.createHandlerForCurrentLooper(); - - ColorInfo inputColorInfo = - sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo) - ? sourceFormat.colorInfo - : ColorInfo.SDR_BT709_LIMITED; - ColorInfo outputColorInfo = inputColorInfo; - if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { - // SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ. - outputColorInfo = - inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build(); - } - - @SuppressWarnings("nullness:assignment") - @Initialized - VideoSinkImpl thisRef = this; - PreviewingVideoGraph videoGraph = - previewingVideoGraphFactory.create( - context, - inputColorInfo, - outputColorInfo, - DebugViewProvider.NONE, - /* listener= */ thisRef, - /* listenerExecutor= */ handler::post, - /* compositionEffects= */ ImmutableList.of(), - /* initialTimestampOffsetUs= */ 0); int videoGraphInputId = videoGraph.registerInput(); videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId); - if (currentSurfaceAndSize != null) { - Size outputSurfaceSize = currentSurfaceAndSize.second; - videoGraph.setOutputSurfaceInfo( - new SurfaceInfo( - currentSurfaceAndSize.first, - outputSurfaceSize.getWidth(), - outputSurfaceSize.getHeight())); - } videoEffects = new ArrayList<>(); - // MediaCodec applies rotation after API 21 + // MediaCodec applies rotation after API 21. rotationEffect = Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0 ? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees) : null; - lastFramePresentationTimeUs = C.TIME_UNSET; + finalBufferPresentationTimeUs = C.TIME_UNSET; + lastBufferPresentationTimeUs = C.TIME_UNSET; } // VideoSink impl @@ -296,27 +428,25 @@ public VideoSinkImpl( @Override public void flush() { videoFrameProcessor.flush(); - processedFramesBufferTimestampsUs.clear(); - streamOffsets.clear(); - handler.removeCallbacksAndMessages(/* token= */ null); - videoFrameReleaseControl.reset(); - lastFramePresentationTimeUs = C.TIME_UNSET; hasRegisteredFirstInputStream = false; - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; - } + finalBufferPresentationTimeUs = C.TIME_UNSET; + lastBufferPresentationTimeUs = C.TIME_UNSET; + compositingVideoSinkProvider.flush(); + // Don't change input stream offset or reset the pending input stream offset change so that + // it's announced with the next input frame. + // Don't reset pendingInputStreamBufferPresentationTimeUs because it's not guaranteed to + // receive a new input stream after seeking. } @Override public boolean isReady() { - return videoFrameReleaseControl.isReady(/* rendererReady= */ true); + return compositingVideoSinkProvider.isReady(); } @Override public boolean isEnded() { - return releasedLastFrame; + return finalBufferPresentationTimeUs != C.TIME_UNSET + && compositingVideoSinkProvider.hasReleasedFrame(finalBufferPresentationTimeUs); } @Override @@ -329,29 +459,20 @@ public void registerInputStream(@InputType int inputType, Format format) { if (!hasRegisteredFirstInputStream) { maybeRegisterInputStream(); hasRegisteredFirstInputStream = true; - // If an input stream registration is pending and seek to another MediaItem, execution - // reaches here before registerInputFrame(), resetting inputStreamRegistrationPending to + // If an input stream registration is pending and seek causes a format change, execution + // reaches here before registerInputFrame(). Reset pendingInputStreamTimestampUs to // avoid registering the same input stream again in registerInputFrame(). - inputStreamRegistrationPending = false; + pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET; } else { - inputStreamRegistrationPending = true; - } - - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; + // If we reach this point, we must have registered at least one frame for processing. + checkState(lastBufferPresentationTimeUs != C.TIME_UNSET); + pendingInputStreamBufferPresentationTimeUs = lastBufferPresentationTimeUs; } } @Override public void setListener(Listener listener, Executor executor) { - if (Util.areEqual(this.listener, listener)) { - checkState(Util.areEqual(listenerExecutor, executor)); - return; - } - this.listener = listener; - this.listenerExecutor = executor; + compositingVideoSinkProvider.setListener(listener, executor); } @Override @@ -370,12 +491,11 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame // An input stream is fully decoded, wait until all of its frames are released before queueing // input frame from the next input stream. - if (inputStreamRegistrationPending) { - if (lastFramePresentationTimeUs == C.TIME_UNSET) { - // A seek took place after signaling a new input stream, but the input stream is yet to be - // registered. + if (pendingInputStreamBufferPresentationTimeUs != C.TIME_UNSET) { + if (compositingVideoSinkProvider.hasReleasedFrame( + pendingInputStreamBufferPresentationTimeUs)) { maybeRegisterInputStream(); - inputStreamRegistrationPending = false; + pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET; } else { return C.TIME_UNSET; } @@ -388,7 +508,6 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame if (!videoFrameProcessor.registerInputFrame()) { return C.TIME_UNSET; } - lastFramePresentationTimeUs = framePresentationTimeUs; // The sink takes in frames with monotonically increasing, non-offset frame // timestamps. That is, with two ten-second long videos, the first frame of the second video // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the @@ -397,12 +516,14 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs; if (pendingInputStreamOffsetChange) { - streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs); + compositingVideoSinkProvider.onStreamOffsetChange( + /* bufferPresentationTimeUs= */ bufferPresentationTimeUs, + /* streamOffsetUs= */ inputStreamOffsetUs); pendingInputStreamOffsetChange = false; } + lastBufferPresentationTimeUs = bufferPresentationTimeUs; if (isLastFrame) { - registeredLastFrame = true; - lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs; + finalBufferPresentationTimeUs = bufferPresentationTimeUs; } return bufferPresentationTimeUs * 1000; } @@ -414,130 +535,27 @@ public boolean queueBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsets @Override public void render(long positionUs, long elapsedRealtimeUs) throws VideoSinkException { - while (!processedFramesBufferTimestampsUs.isEmpty()) { - long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element(); - // check whether this buffer comes with a new stream offset. - if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) { - videoFrameReleaseControl.onProcessedStreamChange(); - } - long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; - boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1; - @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction; - try { - frameReleaseAction = - videoFrameReleaseControl.getFrameReleaseAction( - bufferPresentationTimeUs, - positionUs, - elapsedRealtimeUs, - outputStreamOffsetUs, - isLastFrame, - videoFrameReleaseInfo); - } catch (ExoPlaybackException e) { - throw new VideoSinkException( - e, - new Format.Builder() - .setSampleMimeType(MimeTypes.VIDEO_RAW) - .setWidth(processedFrameSize.width) - .setHeight(processedFrameSize.height) - .build()); - } - switch (frameReleaseAction) { - case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: - return; - case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: - case VideoFrameReleaseControl.FRAME_RELEASE_DROP: - dropFrame(isLastFrame); - break; - case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE: - // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush - // VideoFrameProcessor input frames in this case. - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - break; - case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: - case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: - renderFrame( - framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame); - break; - default: - throw new IllegalStateException(String.valueOf(frameReleaseAction)); - } - if (framePresentationTimeUs == lastFramePresentationTimeUs - && inputStreamRegistrationPending) { - maybeRegisterInputStream(); - inputStreamRegistrationPending = false; - } - } - } - - @Override - public void setPlaybackSpeed(float speed) { - checkArgument(speed >= 0.0); - videoFrameReleaseControl.setPlaybackSpeed(speed); - } - - // VideoGraph.Listener methods - - @Override - public void onOutputSizeChanged(int width, int height) { - VideoSize newVideoSize = new VideoSize(width, height); - if (!processedFrameSize.equals(newVideoSize)) { - processedFrameSize = newVideoSize; - pendingVideoSizeChange = true; - } - } - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (pendingVideoSizeChange) { - videoSizeChanges.add(presentationTimeUs, processedFrameSize); - pendingVideoSizeChange = false; - } - if (registeredLastFrame) { - checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); - } - processedFramesBufferTimestampsUs.add(presentationTimeUs); - // TODO b/257464707 - Support extensively modified media. - if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { - processedLastFrame = true; - } - } - - @Override - public void onError(VideoFrameProcessingException exception) { - if (listener == null || listenerExecutor == null) { - return; + try { + compositingVideoSinkProvider.render(positionUs, elapsedRealtimeUs); + } catch (ExoPlaybackException e) { + throw new VideoSinkException( + e, inputFormat != null ? inputFormat : new Format.Builder().build()); } - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onError( - /* videoSink= */ this, - new VideoSink.VideoSinkException( - exception, - new Format.Builder() - .setSampleMimeType(MimeTypes.VIDEO_RAW) - .setWidth(processedFrameSize.width) - .setHeight(processedFrameSize.height) - .build())); - } - }); } @Override - public void onEnded(long finalFramePresentationTimeUs) { - throw new IllegalStateException(); + public void setPlaybackSpeed(@FloatRange(from = 0, fromInclusive = false) float speed) { + compositingVideoSinkProvider.setPlaybackSpeed(speed); } // Other methods + /** Releases the video sink. */ public void release() { videoFrameProcessor.release(); - handler.removeCallbacksAndMessages(/* token= */ null); - streamOffsets.clear(); - processedFramesBufferTimestampsUs.clear(); } - /** Sets the {@linkplain Effect video effects} to apply immediately. */ + /** Sets the {@linkplain Effect video effects}. */ public void setVideoEffects(List videoEffects) { setPendingVideoEffects(videoEffects); maybeRegisterInputStream(); @@ -552,16 +570,12 @@ public void setPendingVideoEffects(List videoEffects) { this.videoEffects.addAll(videoEffects); } + /** Sets the stream offset, in micro seconds. */ public void setStreamOffsetUs(long streamOffsetUs) { pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs; inputStreamOffsetUs = streamOffsetUs; } - public void setVideoFrameMetadataListener( - VideoFrameMetadataListener videoFrameMetadataListener) { - this.videoFrameMetadataListener = videoFrameMetadataListener; - } - private void maybeRegisterInputStream() { if (inputFormat == null) { return; @@ -581,110 +595,6 @@ private void maybeRegisterInputStream() { .build()); } - /** - * Sets the output surface info. - * - * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs. - * @param outputResolution The {@link Size} of the output resolution. - */ - public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { - if (currentSurfaceAndSize != null - && currentSurfaceAndSize.first.equals(outputSurface) - && currentSurfaceAndSize.second.equals(outputResolution)) { - return; - } - videoFrameReleaseControl.setOutputSurface(outputSurface); - currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); - videoFrameProcessor.setOutputSurfaceInfo( - new SurfaceInfo( - outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); - } - - /** Clears the output surface info. */ - public void clearOutputSurfaceInfo() { - videoFrameProcessor.setOutputSurfaceInfo(null); - currentSurfaceAndSize = null; - } - - private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) { - boolean updatedOffset = false; - @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs); - if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) { - outputStreamOffsetUs = newOutputStreamOffsetUs; - updatedOffset = true; - } - return updatedOffset; - } - - private void dropFrame(boolean isLastFrame) { - if (listenerExecutor != null) { - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onFrameDropped(this); - } - }); - } - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - } - - private void renderFrame( - long framePresentationTimeUs, - long bufferPresentationTimeUs, - @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction, - boolean isLastFrame) { - if (videoFrameMetadataListener != null) { - videoFrameMetadataListener.onVideoFrameAboutToBeRendered( - framePresentationTimeUs, - frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY - ? Clock.DEFAULT.nanoTime() - : videoFrameReleaseInfo.getReleaseTimeNs(), - checkNotNull(inputFormat), - /* mediaFormat= */ null); - } - if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) { - listenerExecutor.execute( - () -> { - if (listener != null) { - listener.onFirstFrameRendered(/* videoSink= */ this); - } - }); - } - releaseProcessedFrameInternal( - frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY - ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY - : videoFrameReleaseInfo.getReleaseTimeNs(), - isLastFrame); - - maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); - } - - private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { - videoFrameProcessor.renderOutputFrame(releaseTimeNs); - processedFramesBufferTimestampsUs.remove(); - if (isLastFrame) { - releasedLastFrame = true; - } - } - - private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) { - if (onVideoSizeChangedCalled || listener == null) { - return; - } - - @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs); - if (videoSize == null) { - return; - } - - if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) { - reportedVideoSize = videoSize; - checkNotNull(listenerExecutor) - .execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize)); - } - onVideoSizeChangedCalled = true; - } - private static final class ScaleAndRotateAccessor { private static @MonotonicNonNull Constructor scaleAndRotateTransformationBuilderConstructor; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 3d494e01241..836be4fe478 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -56,6 +56,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.Size; @@ -393,7 +394,6 @@ public MediaCodecVideoRenderer( assumedMinimumCodecOperatingRate); this.maxDroppedFramesToNotify = maxDroppedFramesToNotify; this.context = context.getApplicationContext(); - @SuppressWarnings("nullness:assignment") VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this; videoFrameReleaseControl = @@ -617,7 +617,9 @@ public static boolean doesDisplaySupportDolbyVision(Context context) { @Override protected void onInit() { super.onInit(); - videoFrameReleaseControl.setClock(getClock()); + Clock clock = getClock(); + videoFrameReleaseControl.setClock(clock); + videoSinkProvider.setClock(clock); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java new file mode 100644 index 00000000000..3824ac6eb28 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -0,0 +1,276 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import androidx.annotation.FloatRange; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoSize; +import androidx.media3.common.util.LongArrayQueue; +import androidx.media3.common.util.TimedValueQueue; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.ExoPlaybackException; + +/** Controls rendering of video frames. */ +/* package */ final class VideoFrameRenderControl { + + /** Receives frames from a {@link VideoFrameRenderControl}. */ + interface FrameRenderer { + + /** + * Called when the {@link VideoSize} changes. This method is called before the frame that + * changes the {@link VideoSize} is passed for render. + */ + void onVideoSizeChanged(VideoSize videoSize); + + /** + * Called to release the {@linkplain + * VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is + * available for rendering}. + * + * @param renderTimeNs The specific time, in nano seconds, that this frame should be rendered or + * {@link VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} if the frame needs to be + * rendered immediately. + * @param presentationTimeUs The frame's presentation time, in microseconds, which was announced + * with {@link VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)}. + * @param streamOffsetUs The stream offset, in microseconds, that is associated with this frame. + * @param isFirstFrame Whether this is the first frame of the stream. + */ + void renderFrame( + long renderTimeNs, long presentationTimeUs, long streamOffsetUs, boolean isFirstFrame); + + /** + * Called to drop the {@linkplain + * VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is + * available for rendering}. + */ + void dropFrame(); + } + + private final FrameRenderer frameRenderer; + private final VideoFrameReleaseControl videoFrameReleaseControl; + private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo; + private final TimedValueQueue videoSizeChanges; + private final TimedValueQueue streamOffsets; + private final LongArrayQueue presentationTimestampsUs; + + /** + * Stores a video size that is announced with {@link #onOutputSizeChanged(int, int)} until an + * output frame is made available. Once the next frame arrives, we associate the frame's timestamp + * with the video size change in {@link #videoSizeChanges} and clear this field. + */ + @Nullable private VideoSize pendingOutputVideoSize; + + private VideoSize reportedVideoSize; + private long outputStreamOffsetUs; + // TODO b/292111083 - Remove the field and trigger the callback on every video size change. + private boolean reportedVideoSizeChange; + private long lastPresentationTimeUs; + + /** Creates an instance. */ + public VideoFrameRenderControl( + FrameRenderer frameRenderer, VideoFrameReleaseControl videoFrameReleaseControl) { + this.frameRenderer = frameRenderer; + this.videoFrameReleaseControl = videoFrameReleaseControl; + videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo(); + videoSizeChanges = new TimedValueQueue<>(); + streamOffsets = new TimedValueQueue<>(); + presentationTimestampsUs = new LongArrayQueue(); + reportedVideoSize = VideoSize.UNKNOWN; + lastPresentationTimeUs = C.TIME_UNSET; + } + + /** Flushes the renderer. */ + public void flush() { + presentationTimestampsUs.clear(); + lastPresentationTimeUs = C.TIME_UNSET; + if (streamOffsets.size() > 0) { + // There is a pending streaming offset change. If seeking within the same stream, keep the + // pending offset with timestamp zero ensures the offset is applied on the frames after + // flushing. Otherwise if seeking to another stream, a new offset will be set before a new + // frame arrives so we'll be able to apply the new offset. + long lastStreamOffset = getLastAndClear(streamOffsets); + streamOffsets.add(/* timestamp= */ 0, lastStreamOffset); + } + if (pendingOutputVideoSize == null) { + if (videoSizeChanges.size() > 0) { + // Do not clear the last pending video size, we still want to report the size change after a + // flush. If after the flush, a new video size is announced, it will overwrite + // pendingOutputVideoSize. When the next frame is available for rendering, we will announce + // pendingOutputVideoSize. + pendingOutputVideoSize = getLastAndClear(videoSizeChanges); + } + } else { + // we keep the latest value of pendingOutputVideoSize + videoSizeChanges.clear(); + } + // Do not clear reportedVideoSizeChange because we report a video size change at most once + // (b/292111083). + } + + /** Returns whether the renderer is ready. */ + public boolean isReady() { + return videoFrameReleaseControl.isReady(/* rendererReady= */ true); + } + + /** + * Returns whether the renderer has released a frame after a specific presentation timestamp. + * + * @param presentationTimeUs The requested timestamp, in microseconds. + * @return Whether the renderer has released a frame with a timestamp greater than or equal to + * {@code presentationTimeUs}. + */ + public boolean hasReleasedFrame(long presentationTimeUs) { + return lastPresentationTimeUs != C.TIME_UNSET && lastPresentationTimeUs >= presentationTimeUs; + } + + /** Sets the playback speed. */ + public void setPlaybackSpeed(@FloatRange(from = 0, fromInclusive = false) float speed) { + checkArgument(speed > 0); + videoFrameReleaseControl.setPlaybackSpeed(speed); + } + + /** + * Incrementally renders available video frames. + * + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * taken approximately at the time the playback position was {@code positionUs}. + */ + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + while (!presentationTimestampsUs.isEmpty()) { + long presentationTimeUs = presentationTimestampsUs.element(); + // Check whether this buffer comes with a new stream offset. + if (maybeUpdateOutputStreamOffset(presentationTimeUs)) { + videoFrameReleaseControl.onProcessedStreamChange(); + } + @VideoFrameReleaseControl.FrameReleaseAction + int frameReleaseAction = + videoFrameReleaseControl.getFrameReleaseAction( + presentationTimeUs, + positionUs, + elapsedRealtimeUs, + outputStreamOffsetUs, + /* isLastFrame= */ false, + videoFrameReleaseInfo); + switch (frameReleaseAction) { + case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER: + return; + case VideoFrameReleaseControl.FRAME_RELEASE_SKIP: + case VideoFrameReleaseControl.FRAME_RELEASE_DROP: + case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE: + // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush + // VideoGraph input frames in this case. + lastPresentationTimeUs = presentationTimeUs; + dropFrame(); + break; + case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY: + case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED: + lastPresentationTimeUs = presentationTimeUs; + renderFrame( + /* shouldRenderImmediately= */ frameReleaseAction + == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY); + break; + default: + throw new IllegalStateException(String.valueOf(frameReleaseAction)); + } + } + } + + /** Called when the size of the available frames has changed. */ + public void onOutputSizeChanged(int width, int height) { + VideoSize newVideoSize = new VideoSize(width, height); + if (!Util.areEqual(pendingOutputVideoSize, newVideoSize)) { + pendingOutputVideoSize = newVideoSize; + } + } + + /** + * Called when a frame is available for rendering. + * + * @param presentationTimeUs The frame's presentation timestamp, in microseconds. + */ + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (pendingOutputVideoSize != null) { + videoSizeChanges.add(presentationTimeUs, pendingOutputVideoSize); + pendingOutputVideoSize = null; + } + presentationTimestampsUs.add(presentationTimeUs); + // TODO b/257464707 - Support extensively modified media. + } + + public void onStreamOffsetChange(long presentationTimeUs, long streamOffsetUs) { + streamOffsets.add(presentationTimeUs, streamOffsetUs); + } + + private void dropFrame() { + checkStateNotNull(presentationTimestampsUs.remove()); + frameRenderer.dropFrame(); + } + + private void renderFrame(boolean shouldRenderImmediately) { + long presentationTimeUs = checkStateNotNull(presentationTimestampsUs.remove()); + + boolean videoSizeUpdated = maybeUpdateVideoSize(presentationTimeUs); + if (videoSizeUpdated && !reportedVideoSizeChange) { + frameRenderer.onVideoSizeChanged(reportedVideoSize); + reportedVideoSizeChange = true; + } + long renderTimeNs = + shouldRenderImmediately + ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY + : videoFrameReleaseInfo.getReleaseTimeNs(); + frameRenderer.renderFrame( + renderTimeNs, + presentationTimeUs, + outputStreamOffsetUs, + videoFrameReleaseControl.onFrameReleasedIsFirstFrame()); + } + + private boolean maybeUpdateOutputStreamOffset(long presentationTimeUs) { + @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(presentationTimeUs); + if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) { + outputStreamOffsetUs = newOutputStreamOffsetUs; + return true; + } + return false; + } + + private boolean maybeUpdateVideoSize(long presentationTimeUs) { + @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(presentationTimeUs); + if (videoSize == null) { + return false; + } + if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) { + reportedVideoSize = videoSize; + return true; + } + return false; + } + + private static T getLastAndClear(TimedValueQueue queue) { + checkArgument(queue.size() > 0); + while (queue.size() > 1) { + queue.pollFirst(); + } + return checkNotNull(queue.pollFirst()); + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 0d4ab5f2f52..a80922fa3b4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -58,11 +58,30 @@ interface Listener { /** Called when the sink dropped a frame. */ void onFrameDropped(VideoSink videoSink); - /** Called when the output video size changed. */ + /** + * Called before a frame is rendered for the first time since setting the surface, and each time + * there's a change in the size, rotation or pixel aspect ratio of the video being rendered. + */ void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize); /** Called when the {@link VideoSink} encountered an error. */ void onError(VideoSink videoSink, VideoSinkException videoSinkException); + + /** A no-op listener implementation. */ + Listener NO_OP = + new Listener() { + @Override + public void onFirstFrameRendered(VideoSink videoSink) {} + + @Override + public void onFrameDropped(VideoSink videoSink) {} + + @Override + public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) {} + + @Override + public void onError(VideoSink videoSink, VideoSinkException videoSinkException) {} + }; } /** diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java index f29f5483977..e0808df4d26 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java @@ -19,6 +19,7 @@ import android.view.Surface; import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.util.Clock; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import java.util.List; @@ -78,4 +79,11 @@ public interface VideoSinkProvider { /** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */ void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener); + + /** + * Sets the {@link Clock} that the provider should use internally. + * + *

Must be called before the sink provider is {@linkplain #initialize(Format) initialized}. + */ + void setClock(Clock clock); } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java new file mode 100644 index 00000000000..92dda8db51d --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java @@ -0,0 +1,372 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; + +import androidx.media3.common.VideoSize; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.test.utils.FakeClock; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.Mockito; + +/** Tests for {@link VideoFrameRenderControl}. */ +@RunWith(AndroidJUnit4.class) +public class VideoFrameRenderControlTest { + + private static final int VIDEO_WIDTH = 640; + private static final int VIDEO_HEIGHT = 480; + + @Test + public void isReady_afterInstantiation_returnsFalse() { + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl( + mock(VideoFrameRenderControl.FrameRenderer.class), createVideoFrameReleaseControl()); + + assertThat(videoFrameRenderControl.isReady()).isFalse(); + } + + @Test + public void releaseFirstFrame() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + assertThat(videoFrameRenderControl.isReady()).isTrue(); + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(0L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + } + + @Test + public void releaseFirstAndSecondFrame() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000); + + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + assertThat(videoFrameRenderControl.isReady()).isTrue(); + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + // First frame. + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(0L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + inOrder.verifyNoMoreInteractions(); + + // 5 seconds pass + clock.advanceTime(/* timeDiffMs= */ 5); + videoFrameRenderControl.render(/* positionUs= */ 5_000, /* elapsedRealtimeUs= */ 5_000); + + // Second frame + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(10_000L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(false)); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void renderFrames_withStreamOffsetSetChange_firstFrameAgain() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + videoFrameReleaseControl.setClock(clock); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onStreamOffsetChange( + /* presentationTimeUs= */ 0, /* streamOffsetUs= */ 10_000); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + assertThat(videoFrameRenderControl.isReady()).isTrue(); + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + // First frame has the first stream offset. + inOrder.verify(frameRenderer).renderFrame(anyLong(), eq(0L), eq(10_000L), eq(true)); + inOrder.verifyNoMoreInteractions(); + + // 10 milliseconds pass + clock.advanceTime(/* timeDiffMs= */ 10); + videoFrameRenderControl.onStreamOffsetChange( + /* presentationTimeUs= */ 10_000, /* streamOffsetUs= */ 20_000); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000); + videoFrameRenderControl.render(/* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0); + + // Second frame has the second stream offset and it is also a first frame. + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(10_000L), + /* streamOffsetUs= */ eq(20_000L), + /* isFirstFrame= */ eq(true)); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void dropFrames() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false); + VideoFrameReleaseControl videoFrameReleaseControl = + createVideoFrameReleaseControl( + new TestFrameTimingEvaluator( + /* shouldForceReleaseFrames= */ false, + /* shouldDropFrames= */ true, + /* shouldIgnoreFrames= */ false)); + videoFrameReleaseControl.setClock(clock); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + // First frame was rendered because the fist frame is force released. + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(0L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + inOrder.verifyNoMoreInteractions(); + + clock.advanceTime(/* timeDiffMs= */ 100); + videoFrameRenderControl.render(/* positionUs= */ 100_000, /* elapsedRealtimeUs= */ 100_000); + + // Second frame was dropped. + inOrder.verify(frameRenderer).dropFrame(); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void flush_removesAvailableFramesForRendering_doesNotFlushOnVideoSizeChange() + throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameReleaseControl.onStarted(); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.flush(); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder.verifyNoMoreInteractions(); + + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + // First frame was rendered with pending video size change. + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(10_000L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void hasReleasedFrame_noFrameReleased_returnsFalse() { + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl( + mock(VideoFrameRenderControl.FrameRenderer.class), videoFrameReleaseControl); + + assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse(); + } + + @Test + public void hasReleasedFrame_frameIsReleased_returnsTrue() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(0L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isTrue(); + } + + @Test + public void hasReleasedFrame_frameIsReleasedAndFlushed_returnsFalse() throws Exception { + VideoFrameRenderControl.FrameRenderer frameRenderer = + mock(VideoFrameRenderControl.FrameRenderer.class); + VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl(); + VideoFrameRenderControl videoFrameRenderControl = + new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl); + + videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true); + videoFrameRenderControl.onOutputSizeChanged( + /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT); + videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0); + videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + + InOrder inOrder = Mockito.inOrder(frameRenderer); + inOrder + .verify(frameRenderer) + .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT)); + inOrder + .verify(frameRenderer) + .renderFrame( + /* renderTimeNs= */ anyLong(), + /* presentationTimeUs= */ eq(0L), + /* streamOffsetUs= */ eq(0L), + /* isFirstFrame= */ eq(true)); + + videoFrameRenderControl.flush(); + + assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse(); + } + + private static VideoFrameReleaseControl createVideoFrameReleaseControl() { + return createVideoFrameReleaseControl( + new TestFrameTimingEvaluator( + /* shouldForceReleaseFrames= */ false, + /* shouldDropFrames= */ false, + /* shouldIgnoreFrames= */ false)); + } + + private static VideoFrameReleaseControl createVideoFrameReleaseControl( + VideoFrameReleaseControl.FrameTimingEvaluator frameTimingEvaluator) { + return new VideoFrameReleaseControl( + ApplicationProvider.getApplicationContext(), + frameTimingEvaluator, + /* allowedJoiningTimeMs= */ 0); + } + + private static class TestFrameTimingEvaluator + implements VideoFrameReleaseControl.FrameTimingEvaluator { + private final boolean shouldForceReleaseFrames; + private final boolean shouldDropFrames; + private final boolean shouldIgnoreFrames; + + public TestFrameTimingEvaluator( + boolean shouldForceReleaseFrames, boolean shouldDropFrames, boolean shouldIgnoreFrames) { + this.shouldForceReleaseFrames = shouldForceReleaseFrames; + this.shouldDropFrames = shouldDropFrames; + this.shouldIgnoreFrames = shouldIgnoreFrames; + } + + @Override + public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { + return shouldForceReleaseFrames; + } + + @Override + public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return shouldDropFrames; + } + + @Override + public boolean shouldIgnoreFrame( + long earlyUs, + long positionUs, + long elapsedRealtimeUs, + boolean isLastFrame, + boolean treatDroppedBuffersAsSkipped) + throws ExoPlaybackException { + return shouldIgnoreFrames; + } + } +}