diff --git a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java
index eb9c8bb08b4..16497da7d9d 100644
--- a/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java
+++ b/libraries/common/src/main/java/androidx/media3/common/PreviewingVideoGraph.java
@@ -53,4 +53,22 @@ PreviewingVideoGraph create(
long initialTimestampOffsetUs)
throws VideoFrameProcessingException;
}
+
+ /**
+ * Renders the oldest unrendered output frame that has become {@linkplain
+ * Listener#onOutputFrameAvailableForRendering(long) available for rendering} at the given {@code
+ * renderTimeNs}.
+ *
+ *
This will either render the output frame to the {@linkplain #setOutputSurfaceInfo output
+ * surface}, or drop the frame, per {@code renderTimeNs}.
+ *
+ *
The {@code renderTimeNs} may be passed to {@link
+ * android.opengl.EGLExt#eglPresentationTimeANDROID} depending on the implementation.
+ *
+ * @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can
+ * be before or after the current system time. Use {@link
+ * VideoFrameProcessor#DROP_OUTPUT_FRAME} to drop the frame, or {@link
+ * VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately.
+ */
+ void renderOutputFrame(long renderTimeNs);
}
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java
index f0568edfe77..fcaa96e8945 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java
@@ -98,4 +98,9 @@ private PreviewingSingleInputVideoGraph(
presentation,
initialTimestampOffsetUs);
}
+
+ @Override
+ public void renderOutputFrame(long renderTimeNs) {
+ getProcessor(SINGLE_INPUT_INDEX).renderOutputFrame(renderTimeNs);
+ }
}
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java
index 15fa3f1e116..bcf36638e09 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java
@@ -31,6 +31,7 @@
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.util.UnstableApi;
+import com.google.common.util.concurrent.MoreExecutors;
import java.util.List;
import java.util.concurrent.Executor;
@@ -54,6 +55,7 @@ public abstract class SingleInputVideoGraph implements VideoGraph {
@Nullable private final Presentation presentation;
@Nullable private VideoFrameProcessor videoFrameProcessor;
+ private boolean isEnded;
private boolean released;
private volatile boolean hasProducedFrameWithTimestampZero;
@@ -112,7 +114,7 @@ public int registerInput() throws VideoFrameProcessingException {
inputColorInfo,
outputColorInfo,
renderFramesAutomatically,
- listenerExecutor,
+ /* listenerExecutor= */ MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
private long lastProcessedFramePresentationTimeUs;
@@ -129,6 +131,12 @@ public void onOutputSizeChanged(int width, int height) {
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+ if (isEnded) {
+ onError(
+ new VideoFrameProcessingException(
+ "onOutputFrameAvailableForRendering() received after onEnded()"));
+ return;
+ }
// Frames are rendered automatically.
if (presentationTimeUs == 0) {
hasProducedFrameWithTimestampZero = true;
@@ -145,7 +153,13 @@ public void onError(VideoFrameProcessingException exception) {
@Override
public void onEnded() {
- listener.onEnded(lastProcessedFramePresentationTimeUs);
+ if (isEnded) {
+ onError(new VideoFrameProcessingException("onEnded() received multiple times"));
+ return;
+ }
+ isEnded = true;
+ listenerExecutor.execute(
+ () -> listener.onEnded(lastProcessedFramePresentationTimeUs));
}
});
return SINGLE_INPUT_INDEX;
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
index 120535476d0..31e83cd4788 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
@@ -15,16 +15,16 @@
*/
package androidx.media3.exoplayer.video;
-import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Bitmap;
-import android.os.Handler;
+import android.os.Looper;
import android.util.Pair;
import android.view.Surface;
+import androidx.annotation.FloatRange;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
@@ -41,9 +41,8 @@
import androidx.media3.common.VideoGraph;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
-import androidx.media3.common.util.LongArrayQueue;
+import androidx.media3.common.util.HandlerWrapper;
import androidx.media3.common.util.Size;
-import androidx.media3.common.util.TimedValueQueue;
import androidx.media3.common.util.TimestampIterator;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
@@ -53,6 +52,7 @@
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import java.util.concurrent.Executor;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
@@ -60,15 +60,27 @@
/** Handles composition of video sinks. */
@UnstableApi
-/* package */ final class CompositingVideoSinkProvider implements VideoSinkProvider {
+/* package */ final class CompositingVideoSinkProvider
+ implements VideoSinkProvider, VideoGraph.Listener, VideoFrameRenderControl.FrameRenderer {
+
+ private static final Executor NO_OP_EXECUTOR = runnable -> {};
private final Context context;
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
private final VideoFrameReleaseControl videoFrameReleaseControl;
-
- @Nullable private VideoSinkImpl videoSinkImpl;
- @Nullable private List videoEffects;
- @Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
+ private final VideoFrameRenderControl videoFrameRenderControl;
+
+ private Clock clock;
+ private @MonotonicNonNull Format outputFormat;
+ private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener;
+ private @MonotonicNonNull HandlerWrapper handler;
+ private @MonotonicNonNull PreviewingVideoGraph videoGraph;
+ private @MonotonicNonNull VideoSinkImpl videoSinkImpl;
+ private @MonotonicNonNull List videoEffects;
+ @Nullable private Pair currentSurfaceAndSize;
+ private VideoSink.Listener listener;
+ private Executor listenerExecutor;
+ private int pendingFlushCount;
private boolean released;
/** Creates a new instance. */
@@ -90,24 +102,54 @@ public CompositingVideoSinkProvider(
this.context = context;
this.previewingVideoGraphFactory = previewingVideoGraphFactory;
this.videoFrameReleaseControl = videoFrameReleaseControl;
+ @SuppressWarnings("nullness:assignment")
+ VideoFrameRenderControl.@Initialized FrameRenderer thisRef = this;
+ videoFrameRenderControl =
+ new VideoFrameRenderControl(/* frameRenderer= */ thisRef, videoFrameReleaseControl);
+ clock = Clock.DEFAULT;
+ listener = VideoSink.Listener.NO_OP;
+ listenerExecutor = NO_OP_EXECUTOR;
}
+ // VideoSinkProvider methods
+
@Override
public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException {
checkState(!released && videoSinkImpl == null);
checkStateNotNull(videoEffects);
+ // Lazily initialize the handler here so it's initialized on the playback looper.
+ handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
+
+ ColorInfo inputColorInfo =
+ sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo)
+ ? sourceFormat.colorInfo
+ : ColorInfo.SDR_BT709_LIMITED;
+ ColorInfo outputColorInfo = inputColorInfo;
+ if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
+ // SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ.
+ outputColorInfo =
+ inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build();
+ }
try {
+ @SuppressWarnings("nullness:assignment")
+ VideoGraph.@Initialized Listener thisRef = this;
+ videoGraph =
+ previewingVideoGraphFactory.create(
+ context,
+ inputColorInfo,
+ outputColorInfo,
+ DebugViewProvider.NONE,
+ /* listener= */ thisRef,
+ /* listenerExecutor= */ handler::post,
+ /* compositionEffects= */ ImmutableList.of(),
+ /* initialTimestampOffsetUs= */ 0);
videoSinkImpl =
new VideoSinkImpl(
- context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat);
+ context, /* compositingVideoSinkProvider= */ this, videoGraph, sourceFormat);
} catch (VideoFrameProcessingException e) {
throw new VideoSink.VideoSinkException(e, sourceFormat);
}
-
- if (videoFrameMetadataListener != null) {
- videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener);
- }
videoSinkImpl.setVideoEffects(checkNotNull(videoEffects));
}
@@ -122,10 +164,16 @@ public void release() {
return;
}
+ if (handler != null) {
+ handler.removeCallbacksAndMessages(/* token= */ null);
+ }
if (videoSinkImpl != null) {
videoSinkImpl.release();
- videoSinkImpl = null;
}
+ if (videoGraph != null) {
+ videoGraph.release();
+ }
+ currentSurfaceAndSize = null;
released = true;
}
@@ -157,138 +205,222 @@ public void setStreamOffsetUs(long streamOffsetUs) {
@Override
public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
- checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution);
+ if (currentSurfaceAndSize != null
+ && currentSurfaceAndSize.first.equals(outputSurface)
+ && currentSurfaceAndSize.second.equals(outputResolution)) {
+ return;
+ }
+ videoFrameReleaseControl.setOutputSurface(outputSurface);
+ currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
+ checkStateNotNull(videoGraph)
+ .setOutputSurfaceInfo(
+ new SurfaceInfo(
+ outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
}
@Override
public void clearOutputSurfaceInfo() {
- checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo();
+ checkStateNotNull(videoGraph).setOutputSurfaceInfo(/* outputSurfaceInfo= */ null);
+ currentSurfaceAndSize = null;
}
@Override
public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) {
this.videoFrameMetadataListener = videoFrameMetadataListener;
- if (isInitialized()) {
- checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener);
+ }
+
+ @Override
+ public void setClock(Clock clock) {
+ checkState(!isInitialized());
+ this.clock = clock;
+ }
+
+ // VideoGraph.Listener
+
+ @Override
+ public void onOutputSizeChanged(int width, int height) {
+ // We forward output size changes to render control even if we are still flushing.
+ videoFrameRenderControl.onOutputSizeChanged(width, height);
+ }
+
+ @Override
+ public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+ if (pendingFlushCount > 0) {
+ // Ignore available frames while the sink provider is flushing
+ return;
+ }
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(presentationTimeUs);
+ }
+
+ @Override
+ public void onEnded(long finalFramePresentationTimeUs) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void onError(VideoFrameProcessingException exception) {
+ VideoSink.Listener currentListener = this.listener;
+ listenerExecutor.execute(
+ () -> {
+ VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
+ currentListener.onError(
+ videoSink,
+ new VideoSink.VideoSinkException(
+ exception, checkStateNotNull(videoSink.inputFormat)));
+ });
+ }
+
+ // FrameRenderer methods
+
+ @Override
+ public void onVideoSizeChanged(VideoSize videoSize) {
+ outputFormat =
+ new Format.Builder()
+ .setWidth(videoSize.width)
+ .setHeight(videoSize.height)
+ .setSampleMimeType(MimeTypes.VIDEO_RAW)
+ .build();
+ VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
+ VideoSink.Listener currentListener = this.listener;
+ listenerExecutor.execute(() -> currentListener.onVideoSizeChanged(videoSink, videoSize));
+ }
+
+ @Override
+ public void renderFrame(
+ long renderTimeNs, long bufferPresentationTimeUs, long streamOffsetUs, boolean isFirstFrame) {
+ if (isFirstFrame && listenerExecutor != NO_OP_EXECUTOR) {
+ VideoSinkImpl videoSink = checkStateNotNull(videoSinkImpl);
+ VideoSink.Listener currentListener = this.listener;
+ listenerExecutor.execute(() -> currentListener.onFirstFrameRendered(videoSink));
}
+ if (videoFrameMetadataListener != null) {
+ // TODO b/292111083 - outputFormat is initialized after the first frame is rendered because
+ // onVideoSizeChanged is announced after the first frame is available for rendering.
+ Format format = outputFormat == null ? new Format.Builder().build() : outputFormat;
+ videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
+ /* presentationTimeUs= */ bufferPresentationTimeUs - streamOffsetUs,
+ clock.nanoTime(),
+ format,
+ /* mediaFormat= */ null);
+ }
+ checkStateNotNull(videoGraph).renderOutputFrame(renderTimeNs);
}
- private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener {
+ @Override
+ public void dropFrame() {
+ VideoSink.Listener currentListener = this.listener;
+ listenerExecutor.execute(
+ () -> currentListener.onFrameDropped(checkStateNotNull(videoSinkImpl)));
+ checkStateNotNull(videoGraph).renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME);
+ }
+
+ // Internal methods
+
+ private void setListener(VideoSink.Listener listener, Executor executor) {
+ if (Objects.equals(listener, this.listener)) {
+ checkState(Objects.equals(executor, listenerExecutor));
+ return;
+ }
+ this.listener = listener;
+ this.listenerExecutor = executor;
+ }
+
+ private boolean isReady() {
+ return pendingFlushCount == 0 && videoFrameRenderControl.isReady();
+ }
+
+ private boolean hasReleasedFrame(long presentationTimeUs) {
+ return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs);
+ }
+
+ private void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+ if (pendingFlushCount == 0) {
+ videoFrameRenderControl.render(positionUs, elapsedRealtimeUs);
+ }
+ }
+
+ private void flush() {
+ pendingFlushCount++;
+ // Flush the render control now to ensure it has no data, eg calling isReady() must return false
+ // and
+ // render() should not render any frames.
+ videoFrameRenderControl.flush();
+ // Finish flushing after handling pending video graph callbacks to ensure video size changes
+ // reach the video render control.
+ checkStateNotNull(handler).post(this::flushInternal);
+ }
+
+ private void flushInternal() {
+ pendingFlushCount--;
+ if (pendingFlushCount > 0) {
+ // Another flush has been issued.
+ return;
+ } else if (pendingFlushCount < 0) {
+ throw new IllegalStateException(String.valueOf(pendingFlushCount));
+ }
+ // Flush the render control again.
+ videoFrameRenderControl.flush();
+ }
+
+ private void setPlaybackSpeed(float speed) {
+ videoFrameRenderControl.setPlaybackSpeed(speed);
+ }
+
+ private void onStreamOffsetChange(long bufferPresentationTimeUs, long streamOffsetUs) {
+ videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
+ }
+
+ /** Receives input from an ExoPlayer renderer and forwards it to the video graph. */
+ private static final class VideoSinkImpl implements VideoSink {
private final Context context;
- private final VideoFrameReleaseControl videoFrameReleaseControl;
- private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
+ private final CompositingVideoSinkProvider compositingVideoSinkProvider;
private final VideoFrameProcessor videoFrameProcessor;
- private final LongArrayQueue processedFramesBufferTimestampsUs;
- private final TimedValueQueue streamOffsets;
- private final TimedValueQueue videoSizeChanges;
- private final Handler handler;
private final int videoFrameProcessorMaxPendingFrameCount;
private final ArrayList videoEffects;
@Nullable private final Effect rotationEffect;
- private VideoSink.@MonotonicNonNull Listener listener;
- private @MonotonicNonNull Executor listenerExecutor;
- @Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
@Nullable private Format inputFormat;
- @Nullable private Pair currentSurfaceAndSize;
+ private long inputStreamOffsetUs;
+ private boolean pendingInputStreamOffsetChange;
- /**
- * Whether the last frame of the current stream is decoded and registered to {@link
- * VideoFrameProcessor}.
- */
- private boolean registeredLastFrame;
+ /** The buffer presentation time, in microseconds, of the final frame in the stream. */
+ private long finalBufferPresentationTimeUs;
/**
- * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}.
+ * The buffer presentation timestamp, in microseconds, of the most recently registered frame.
*/
- private boolean processedLastFrame;
+ private long lastBufferPresentationTimeUs;
- /** Whether the last frame of the current stream is released to the output {@link Surface}. */
- private boolean releasedLastFrame;
-
- private long lastCodecBufferPresentationTimestampUs;
- private VideoSize processedFrameSize;
- private VideoSize reportedVideoSize;
- private boolean pendingVideoSizeChange;
- private long inputStreamOffsetUs;
- private boolean pendingInputStreamOffsetChange;
- private long outputStreamOffsetUs;
-
- // TODO b/292111083 - Remove the field and trigger the callback on every video size change.
- private boolean onVideoSizeChangedCalled;
private boolean hasRegisteredFirstInputStream;
- private boolean inputStreamRegistrationPending;
- private long lastFramePresentationTimeUs;
+ private long pendingInputStreamBufferPresentationTimeUs;
/** Creates a new instance. */
public VideoSinkImpl(
Context context,
- PreviewingVideoGraph.Factory previewingVideoGraphFactory,
- VideoFrameReleaseControl videoFrameReleaseControl,
+ CompositingVideoSinkProvider compositingVideoSinkProvider,
+ PreviewingVideoGraph videoGraph,
Format sourceFormat)
throws VideoFrameProcessingException {
this.context = context;
- this.videoFrameReleaseControl = videoFrameReleaseControl;
- videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
- processedFramesBufferTimestampsUs = new LongArrayQueue();
- streamOffsets = new TimedValueQueue<>();
- videoSizeChanges = new TimedValueQueue<>();
+ this.compositingVideoSinkProvider = compositingVideoSinkProvider;
// TODO b/226330223 - Investigate increasing frame count when frame dropping is
// allowed.
// TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed
// reduces decoder timeouts, and consider restoring.
videoFrameProcessorMaxPendingFrameCount =
Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
- lastCodecBufferPresentationTimestampUs = C.TIME_UNSET;
- processedFrameSize = VideoSize.UNKNOWN;
- reportedVideoSize = VideoSize.UNKNOWN;
-
- // Playback thread handler.
- handler = Util.createHandlerForCurrentLooper();
-
- ColorInfo inputColorInfo =
- sourceFormat.colorInfo != null && ColorInfo.isTransferHdr(sourceFormat.colorInfo)
- ? sourceFormat.colorInfo
- : ColorInfo.SDR_BT709_LIMITED;
- ColorInfo outputColorInfo = inputColorInfo;
- if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
- // SurfaceView only supports BT2020 PQ input. Therefore, convert HLG to PQ.
- outputColorInfo =
- inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build();
- }
-
- @SuppressWarnings("nullness:assignment")
- @Initialized
- VideoSinkImpl thisRef = this;
- PreviewingVideoGraph videoGraph =
- previewingVideoGraphFactory.create(
- context,
- inputColorInfo,
- outputColorInfo,
- DebugViewProvider.NONE,
- /* listener= */ thisRef,
- /* listenerExecutor= */ handler::post,
- /* compositionEffects= */ ImmutableList.of(),
- /* initialTimestampOffsetUs= */ 0);
int videoGraphInputId = videoGraph.registerInput();
videoFrameProcessor = videoGraph.getProcessor(videoGraphInputId);
- if (currentSurfaceAndSize != null) {
- Size outputSurfaceSize = currentSurfaceAndSize.second;
- videoGraph.setOutputSurfaceInfo(
- new SurfaceInfo(
- currentSurfaceAndSize.first,
- outputSurfaceSize.getWidth(),
- outputSurfaceSize.getHeight()));
- }
videoEffects = new ArrayList<>();
- // MediaCodec applies rotation after API 21
+ // MediaCodec applies rotation after API 21.
rotationEffect =
Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0
? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees)
: null;
- lastFramePresentationTimeUs = C.TIME_UNSET;
+ finalBufferPresentationTimeUs = C.TIME_UNSET;
+ lastBufferPresentationTimeUs = C.TIME_UNSET;
}
// VideoSink impl
@@ -296,27 +428,25 @@ public VideoSinkImpl(
@Override
public void flush() {
videoFrameProcessor.flush();
- processedFramesBufferTimestampsUs.clear();
- streamOffsets.clear();
- handler.removeCallbacksAndMessages(/* token= */ null);
- videoFrameReleaseControl.reset();
- lastFramePresentationTimeUs = C.TIME_UNSET;
hasRegisteredFirstInputStream = false;
- if (registeredLastFrame) {
- registeredLastFrame = false;
- processedLastFrame = false;
- releasedLastFrame = false;
- }
+ finalBufferPresentationTimeUs = C.TIME_UNSET;
+ lastBufferPresentationTimeUs = C.TIME_UNSET;
+ compositingVideoSinkProvider.flush();
+ // Don't change input stream offset or reset the pending input stream offset change so that
+ // it's announced with the next input frame.
+ // Don't reset pendingInputStreamBufferPresentationTimeUs because it's not guaranteed to
+ // receive a new input stream after seeking.
}
@Override
public boolean isReady() {
- return videoFrameReleaseControl.isReady(/* rendererReady= */ true);
+ return compositingVideoSinkProvider.isReady();
}
@Override
public boolean isEnded() {
- return releasedLastFrame;
+ return finalBufferPresentationTimeUs != C.TIME_UNSET
+ && compositingVideoSinkProvider.hasReleasedFrame(finalBufferPresentationTimeUs);
}
@Override
@@ -329,29 +459,20 @@ public void registerInputStream(@InputType int inputType, Format format) {
if (!hasRegisteredFirstInputStream) {
maybeRegisterInputStream();
hasRegisteredFirstInputStream = true;
- // If an input stream registration is pending and seek to another MediaItem, execution
- // reaches here before registerInputFrame(), resetting inputStreamRegistrationPending to
+ // If an input stream registration is pending and seek causes a format change, execution
+ // reaches here before registerInputFrame(). Reset pendingInputStreamTimestampUs to
// avoid registering the same input stream again in registerInputFrame().
- inputStreamRegistrationPending = false;
+ pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET;
} else {
- inputStreamRegistrationPending = true;
- }
-
- if (registeredLastFrame) {
- registeredLastFrame = false;
- processedLastFrame = false;
- releasedLastFrame = false;
+ // If we reach this point, we must have registered at least one frame for processing.
+ checkState(lastBufferPresentationTimeUs != C.TIME_UNSET);
+ pendingInputStreamBufferPresentationTimeUs = lastBufferPresentationTimeUs;
}
}
@Override
public void setListener(Listener listener, Executor executor) {
- if (Util.areEqual(this.listener, listener)) {
- checkState(Util.areEqual(listenerExecutor, executor));
- return;
- }
- this.listener = listener;
- this.listenerExecutor = executor;
+ compositingVideoSinkProvider.setListener(listener, executor);
}
@Override
@@ -370,12 +491,11 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame
// An input stream is fully decoded, wait until all of its frames are released before queueing
// input frame from the next input stream.
- if (inputStreamRegistrationPending) {
- if (lastFramePresentationTimeUs == C.TIME_UNSET) {
- // A seek took place after signaling a new input stream, but the input stream is yet to be
- // registered.
+ if (pendingInputStreamBufferPresentationTimeUs != C.TIME_UNSET) {
+ if (compositingVideoSinkProvider.hasReleasedFrame(
+ pendingInputStreamBufferPresentationTimeUs)) {
maybeRegisterInputStream();
- inputStreamRegistrationPending = false;
+ pendingInputStreamBufferPresentationTimeUs = C.TIME_UNSET;
} else {
return C.TIME_UNSET;
}
@@ -388,7 +508,6 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame
if (!videoFrameProcessor.registerInputFrame()) {
return C.TIME_UNSET;
}
- lastFramePresentationTimeUs = framePresentationTimeUs;
// The sink takes in frames with monotonically increasing, non-offset frame
// timestamps. That is, with two ten-second long videos, the first frame of the second video
// should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
@@ -397,12 +516,14 @@ public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame
// handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs;
if (pendingInputStreamOffsetChange) {
- streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs);
+ compositingVideoSinkProvider.onStreamOffsetChange(
+ /* bufferPresentationTimeUs= */ bufferPresentationTimeUs,
+ /* streamOffsetUs= */ inputStreamOffsetUs);
pendingInputStreamOffsetChange = false;
}
+ lastBufferPresentationTimeUs = bufferPresentationTimeUs;
if (isLastFrame) {
- registeredLastFrame = true;
- lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs;
+ finalBufferPresentationTimeUs = bufferPresentationTimeUs;
}
return bufferPresentationTimeUs * 1000;
}
@@ -414,130 +535,27 @@ public boolean queueBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsets
@Override
public void render(long positionUs, long elapsedRealtimeUs) throws VideoSinkException {
- while (!processedFramesBufferTimestampsUs.isEmpty()) {
- long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element();
- // check whether this buffer comes with a new stream offset.
- if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) {
- videoFrameReleaseControl.onProcessedStreamChange();
- }
- long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
- boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1;
- @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction;
- try {
- frameReleaseAction =
- videoFrameReleaseControl.getFrameReleaseAction(
- bufferPresentationTimeUs,
- positionUs,
- elapsedRealtimeUs,
- outputStreamOffsetUs,
- isLastFrame,
- videoFrameReleaseInfo);
- } catch (ExoPlaybackException e) {
- throw new VideoSinkException(
- e,
- new Format.Builder()
- .setSampleMimeType(MimeTypes.VIDEO_RAW)
- .setWidth(processedFrameSize.width)
- .setHeight(processedFrameSize.height)
- .build());
- }
- switch (frameReleaseAction) {
- case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER:
- return;
- case VideoFrameReleaseControl.FRAME_RELEASE_SKIP:
- case VideoFrameReleaseControl.FRAME_RELEASE_DROP:
- dropFrame(isLastFrame);
- break;
- case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE:
- // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
- // VideoFrameProcessor input frames in this case.
- releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
- break;
- case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
- case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
- renderFrame(
- framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame);
- break;
- default:
- throw new IllegalStateException(String.valueOf(frameReleaseAction));
- }
- if (framePresentationTimeUs == lastFramePresentationTimeUs
- && inputStreamRegistrationPending) {
- maybeRegisterInputStream();
- inputStreamRegistrationPending = false;
- }
- }
- }
-
- @Override
- public void setPlaybackSpeed(float speed) {
- checkArgument(speed >= 0.0);
- videoFrameReleaseControl.setPlaybackSpeed(speed);
- }
-
- // VideoGraph.Listener methods
-
- @Override
- public void onOutputSizeChanged(int width, int height) {
- VideoSize newVideoSize = new VideoSize(width, height);
- if (!processedFrameSize.equals(newVideoSize)) {
- processedFrameSize = newVideoSize;
- pendingVideoSizeChange = true;
- }
- }
-
- @Override
- public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
- if (pendingVideoSizeChange) {
- videoSizeChanges.add(presentationTimeUs, processedFrameSize);
- pendingVideoSizeChange = false;
- }
- if (registeredLastFrame) {
- checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
- }
- processedFramesBufferTimestampsUs.add(presentationTimeUs);
- // TODO b/257464707 - Support extensively modified media.
- if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
- processedLastFrame = true;
- }
- }
-
- @Override
- public void onError(VideoFrameProcessingException exception) {
- if (listener == null || listenerExecutor == null) {
- return;
+ try {
+ compositingVideoSinkProvider.render(positionUs, elapsedRealtimeUs);
+ } catch (ExoPlaybackException e) {
+ throw new VideoSinkException(
+ e, inputFormat != null ? inputFormat : new Format.Builder().build());
}
- listenerExecutor.execute(
- () -> {
- if (listener != null) {
- listener.onError(
- /* videoSink= */ this,
- new VideoSink.VideoSinkException(
- exception,
- new Format.Builder()
- .setSampleMimeType(MimeTypes.VIDEO_RAW)
- .setWidth(processedFrameSize.width)
- .setHeight(processedFrameSize.height)
- .build()));
- }
- });
}
@Override
- public void onEnded(long finalFramePresentationTimeUs) {
- throw new IllegalStateException();
+ public void setPlaybackSpeed(@FloatRange(from = 0, fromInclusive = false) float speed) {
+ compositingVideoSinkProvider.setPlaybackSpeed(speed);
}
// Other methods
+ /** Releases the video sink. */
public void release() {
videoFrameProcessor.release();
- handler.removeCallbacksAndMessages(/* token= */ null);
- streamOffsets.clear();
- processedFramesBufferTimestampsUs.clear();
}
- /** Sets the {@linkplain Effect video effects} to apply immediately. */
+ /** Sets the {@linkplain Effect video effects}. */
public void setVideoEffects(List videoEffects) {
setPendingVideoEffects(videoEffects);
maybeRegisterInputStream();
@@ -552,16 +570,12 @@ public void setPendingVideoEffects(List videoEffects) {
this.videoEffects.addAll(videoEffects);
}
+ /** Sets the stream offset, in micro seconds. */
public void setStreamOffsetUs(long streamOffsetUs) {
pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs;
inputStreamOffsetUs = streamOffsetUs;
}
- public void setVideoFrameMetadataListener(
- VideoFrameMetadataListener videoFrameMetadataListener) {
- this.videoFrameMetadataListener = videoFrameMetadataListener;
- }
-
private void maybeRegisterInputStream() {
if (inputFormat == null) {
return;
@@ -581,110 +595,6 @@ private void maybeRegisterInputStream() {
.build());
}
- /**
- * Sets the output surface info.
- *
- * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs.
- * @param outputResolution The {@link Size} of the output resolution.
- */
- public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
- if (currentSurfaceAndSize != null
- && currentSurfaceAndSize.first.equals(outputSurface)
- && currentSurfaceAndSize.second.equals(outputResolution)) {
- return;
- }
- videoFrameReleaseControl.setOutputSurface(outputSurface);
- currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
- videoFrameProcessor.setOutputSurfaceInfo(
- new SurfaceInfo(
- outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
- }
-
- /** Clears the output surface info. */
- public void clearOutputSurfaceInfo() {
- videoFrameProcessor.setOutputSurfaceInfo(null);
- currentSurfaceAndSize = null;
- }
-
- private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
- boolean updatedOffset = false;
- @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs);
- if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) {
- outputStreamOffsetUs = newOutputStreamOffsetUs;
- updatedOffset = true;
- }
- return updatedOffset;
- }
-
- private void dropFrame(boolean isLastFrame) {
- if (listenerExecutor != null) {
- listenerExecutor.execute(
- () -> {
- if (listener != null) {
- listener.onFrameDropped(this);
- }
- });
- }
- releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
- }
-
- private void renderFrame(
- long framePresentationTimeUs,
- long bufferPresentationTimeUs,
- @VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction,
- boolean isLastFrame) {
- if (videoFrameMetadataListener != null) {
- videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
- framePresentationTimeUs,
- frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
- ? Clock.DEFAULT.nanoTime()
- : videoFrameReleaseInfo.getReleaseTimeNs(),
- checkNotNull(inputFormat),
- /* mediaFormat= */ null);
- }
- if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) {
- listenerExecutor.execute(
- () -> {
- if (listener != null) {
- listener.onFirstFrameRendered(/* videoSink= */ this);
- }
- });
- }
- releaseProcessedFrameInternal(
- frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
- ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
- : videoFrameReleaseInfo.getReleaseTimeNs(),
- isLastFrame);
-
- maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
- }
-
- private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) {
- videoFrameProcessor.renderOutputFrame(releaseTimeNs);
- processedFramesBufferTimestampsUs.remove();
- if (isLastFrame) {
- releasedLastFrame = true;
- }
- }
-
- private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) {
- if (onVideoSizeChangedCalled || listener == null) {
- return;
- }
-
- @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs);
- if (videoSize == null) {
- return;
- }
-
- if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) {
- reportedVideoSize = videoSize;
- checkNotNull(listenerExecutor)
- .execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize));
- }
- onVideoSizeChangedCalled = true;
- }
-
private static final class ScaleAndRotateAccessor {
private static @MonotonicNonNull Constructor>
scaleAndRotateTransformationBuilderConstructor;
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
index 3d494e01241..836be4fe478 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
@@ -56,6 +56,7 @@
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize;
+import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.Size;
@@ -393,7 +394,6 @@ public MediaCodecVideoRenderer(
assumedMinimumCodecOperatingRate);
this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
this.context = context.getApplicationContext();
-
@SuppressWarnings("nullness:assignment")
VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this;
videoFrameReleaseControl =
@@ -617,7 +617,9 @@ public static boolean doesDisplaySupportDolbyVision(Context context) {
@Override
protected void onInit() {
super.onInit();
- videoFrameReleaseControl.setClock(getClock());
+ Clock clock = getClock();
+ videoFrameReleaseControl.setClock(clock);
+ videoSinkProvider.setClock(clock);
}
@Override
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java
new file mode 100644
index 00000000000..3824ac6eb28
--- /dev/null
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.video;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+
+import androidx.annotation.FloatRange;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.common.VideoFrameProcessor;
+import androidx.media3.common.VideoSize;
+import androidx.media3.common.util.LongArrayQueue;
+import androidx.media3.common.util.TimedValueQueue;
+import androidx.media3.common.util.Util;
+import androidx.media3.exoplayer.ExoPlaybackException;
+
+/** Controls rendering of video frames. */
+/* package */ final class VideoFrameRenderControl {
+
+ /** Receives frames from a {@link VideoFrameRenderControl}. */
+ interface FrameRenderer {
+
+ /**
+ * Called when the {@link VideoSize} changes. This method is called before the frame that
+ * changes the {@link VideoSize} is passed for render.
+ */
+ void onVideoSizeChanged(VideoSize videoSize);
+
+ /**
+ * Called to release the {@linkplain
+ * VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is
+ * available for rendering}.
+ *
+ * @param renderTimeNs The specific time, in nano seconds, that this frame should be rendered or
+ * {@link VideoFrameProcessor#RENDER_OUTPUT_FRAME_IMMEDIATELY} if the frame needs to be
+ * rendered immediately.
+ * @param presentationTimeUs The frame's presentation time, in microseconds, which was announced
+ * with {@link VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)}.
+ * @param streamOffsetUs The stream offset, in microseconds, that is associated with this frame.
+ * @param isFirstFrame Whether this is the first frame of the stream.
+ */
+ void renderFrame(
+ long renderTimeNs, long presentationTimeUs, long streamOffsetUs, boolean isFirstFrame);
+
+ /**
+ * Called to drop the {@linkplain
+ * VideoFrameRenderControl#onOutputFrameAvailableForRendering(long)} oldest frame that is
+ * available for rendering}.
+ */
+ void dropFrame();
+ }
+
+ private final FrameRenderer frameRenderer;
+ private final VideoFrameReleaseControl videoFrameReleaseControl;
+ private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
+ private final TimedValueQueue videoSizeChanges;
+ private final TimedValueQueue streamOffsets;
+ private final LongArrayQueue presentationTimestampsUs;
+
+ /**
+ * Stores a video size that is announced with {@link #onOutputSizeChanged(int, int)} until an
+ * output frame is made available. Once the next frame arrives, we associate the frame's timestamp
+ * with the video size change in {@link #videoSizeChanges} and clear this field.
+ */
+ @Nullable private VideoSize pendingOutputVideoSize;
+
+ private VideoSize reportedVideoSize;
+ private long outputStreamOffsetUs;
+ // TODO b/292111083 - Remove the field and trigger the callback on every video size change.
+ private boolean reportedVideoSizeChange;
+ private long lastPresentationTimeUs;
+
+ /** Creates an instance. */
+ public VideoFrameRenderControl(
+ FrameRenderer frameRenderer, VideoFrameReleaseControl videoFrameReleaseControl) {
+ this.frameRenderer = frameRenderer;
+ this.videoFrameReleaseControl = videoFrameReleaseControl;
+ videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
+ videoSizeChanges = new TimedValueQueue<>();
+ streamOffsets = new TimedValueQueue<>();
+ presentationTimestampsUs = new LongArrayQueue();
+ reportedVideoSize = VideoSize.UNKNOWN;
+ lastPresentationTimeUs = C.TIME_UNSET;
+ }
+
+ /** Flushes the renderer. */
+ public void flush() {
+ presentationTimestampsUs.clear();
+ lastPresentationTimeUs = C.TIME_UNSET;
+ if (streamOffsets.size() > 0) {
+ // There is a pending streaming offset change. If seeking within the same stream, keep the
+ // pending offset with timestamp zero ensures the offset is applied on the frames after
+ // flushing. Otherwise if seeking to another stream, a new offset will be set before a new
+ // frame arrives so we'll be able to apply the new offset.
+ long lastStreamOffset = getLastAndClear(streamOffsets);
+ streamOffsets.add(/* timestamp= */ 0, lastStreamOffset);
+ }
+ if (pendingOutputVideoSize == null) {
+ if (videoSizeChanges.size() > 0) {
+ // Do not clear the last pending video size, we still want to report the size change after a
+ // flush. If after the flush, a new video size is announced, it will overwrite
+ // pendingOutputVideoSize. When the next frame is available for rendering, we will announce
+ // pendingOutputVideoSize.
+ pendingOutputVideoSize = getLastAndClear(videoSizeChanges);
+ }
+ } else {
+ // we keep the latest value of pendingOutputVideoSize
+ videoSizeChanges.clear();
+ }
+ // Do not clear reportedVideoSizeChange because we report a video size change at most once
+ // (b/292111083).
+ }
+
+ /** Returns whether the renderer is ready. */
+ public boolean isReady() {
+ return videoFrameReleaseControl.isReady(/* rendererReady= */ true);
+ }
+
+ /**
+ * Returns whether the renderer has released a frame after a specific presentation timestamp.
+ *
+ * @param presentationTimeUs The requested timestamp, in microseconds.
+ * @return Whether the renderer has released a frame with a timestamp greater than or equal to
+ * {@code presentationTimeUs}.
+ */
+ public boolean hasReleasedFrame(long presentationTimeUs) {
+ return lastPresentationTimeUs != C.TIME_UNSET && lastPresentationTimeUs >= presentationTimeUs;
+ }
+
+ /** Sets the playback speed. */
+ public void setPlaybackSpeed(@FloatRange(from = 0, fromInclusive = false) float speed) {
+ checkArgument(speed > 0);
+ videoFrameReleaseControl.setPlaybackSpeed(speed);
+ }
+
+ /**
+ * Incrementally renders available video frames.
+ *
+ * @param positionUs The current playback position, in microseconds.
+ * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+ * taken approximately at the time the playback position was {@code positionUs}.
+ */
+ public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+ while (!presentationTimestampsUs.isEmpty()) {
+ long presentationTimeUs = presentationTimestampsUs.element();
+ // Check whether this buffer comes with a new stream offset.
+ if (maybeUpdateOutputStreamOffset(presentationTimeUs)) {
+ videoFrameReleaseControl.onProcessedStreamChange();
+ }
+ @VideoFrameReleaseControl.FrameReleaseAction
+ int frameReleaseAction =
+ videoFrameReleaseControl.getFrameReleaseAction(
+ presentationTimeUs,
+ positionUs,
+ elapsedRealtimeUs,
+ outputStreamOffsetUs,
+ /* isLastFrame= */ false,
+ videoFrameReleaseInfo);
+ switch (frameReleaseAction) {
+ case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER:
+ return;
+ case VideoFrameReleaseControl.FRAME_RELEASE_SKIP:
+ case VideoFrameReleaseControl.FRAME_RELEASE_DROP:
+ case VideoFrameReleaseControl.FRAME_RELEASE_IGNORE:
+ // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
+ // VideoGraph input frames in this case.
+ lastPresentationTimeUs = presentationTimeUs;
+ dropFrame();
+ break;
+ case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
+ case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
+ lastPresentationTimeUs = presentationTimeUs;
+ renderFrame(
+ /* shouldRenderImmediately= */ frameReleaseAction
+ == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY);
+ break;
+ default:
+ throw new IllegalStateException(String.valueOf(frameReleaseAction));
+ }
+ }
+ }
+
+ /** Called when the size of the available frames has changed. */
+ public void onOutputSizeChanged(int width, int height) {
+ VideoSize newVideoSize = new VideoSize(width, height);
+ if (!Util.areEqual(pendingOutputVideoSize, newVideoSize)) {
+ pendingOutputVideoSize = newVideoSize;
+ }
+ }
+
+ /**
+ * Called when a frame is available for rendering.
+ *
+ * @param presentationTimeUs The frame's presentation timestamp, in microseconds.
+ */
+ public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
+ if (pendingOutputVideoSize != null) {
+ videoSizeChanges.add(presentationTimeUs, pendingOutputVideoSize);
+ pendingOutputVideoSize = null;
+ }
+ presentationTimestampsUs.add(presentationTimeUs);
+ // TODO b/257464707 - Support extensively modified media.
+ }
+
+ public void onStreamOffsetChange(long presentationTimeUs, long streamOffsetUs) {
+ streamOffsets.add(presentationTimeUs, streamOffsetUs);
+ }
+
+ private void dropFrame() {
+ checkStateNotNull(presentationTimestampsUs.remove());
+ frameRenderer.dropFrame();
+ }
+
+ private void renderFrame(boolean shouldRenderImmediately) {
+ long presentationTimeUs = checkStateNotNull(presentationTimestampsUs.remove());
+
+ boolean videoSizeUpdated = maybeUpdateVideoSize(presentationTimeUs);
+ if (videoSizeUpdated && !reportedVideoSizeChange) {
+ frameRenderer.onVideoSizeChanged(reportedVideoSize);
+ reportedVideoSizeChange = true;
+ }
+ long renderTimeNs =
+ shouldRenderImmediately
+ ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
+ : videoFrameReleaseInfo.getReleaseTimeNs();
+ frameRenderer.renderFrame(
+ renderTimeNs,
+ presentationTimeUs,
+ outputStreamOffsetUs,
+ videoFrameReleaseControl.onFrameReleasedIsFirstFrame());
+ }
+
+ private boolean maybeUpdateOutputStreamOffset(long presentationTimeUs) {
+ @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(presentationTimeUs);
+ if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) {
+ outputStreamOffsetUs = newOutputStreamOffsetUs;
+ return true;
+ }
+ return false;
+ }
+
+ private boolean maybeUpdateVideoSize(long presentationTimeUs) {
+ @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(presentationTimeUs);
+ if (videoSize == null) {
+ return false;
+ }
+ if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) {
+ reportedVideoSize = videoSize;
+ return true;
+ }
+ return false;
+ }
+
+ private static T getLastAndClear(TimedValueQueue queue) {
+ checkArgument(queue.size() > 0);
+ while (queue.size() > 1) {
+ queue.pollFirst();
+ }
+ return checkNotNull(queue.pollFirst());
+ }
+}
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java
index 0d4ab5f2f52..a80922fa3b4 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java
@@ -58,11 +58,30 @@ interface Listener {
/** Called when the sink dropped a frame. */
void onFrameDropped(VideoSink videoSink);
- /** Called when the output video size changed. */
+ /**
+ * Called before a frame is rendered for the first time since setting the surface, and each time
+ * there's a change in the size, rotation or pixel aspect ratio of the video being rendered.
+ */
void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize);
/** Called when the {@link VideoSink} encountered an error. */
void onError(VideoSink videoSink, VideoSinkException videoSinkException);
+
+ /** A no-op listener implementation. */
+ Listener NO_OP =
+ new Listener() {
+ @Override
+ public void onFirstFrameRendered(VideoSink videoSink) {}
+
+ @Override
+ public void onFrameDropped(VideoSink videoSink) {}
+
+ @Override
+ public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) {}
+
+ @Override
+ public void onError(VideoSink videoSink, VideoSinkException videoSinkException) {}
+ };
}
/**
diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java
index f29f5483977..e0808df4d26 100644
--- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java
+++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java
@@ -19,6 +19,7 @@
import android.view.Surface;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
+import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.util.List;
@@ -78,4 +79,11 @@ public interface VideoSinkProvider {
/** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */
void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener);
+
+ /**
+ * Sets the {@link Clock} that the provider should use internally.
+ *
+ * Must be called before the sink provider is {@linkplain #initialize(Format) initialized}.
+ */
+ void setClock(Clock clock);
}
diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java
new file mode 100644
index 00000000000..92dda8db51d
--- /dev/null
+++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/VideoFrameRenderControlTest.java
@@ -0,0 +1,372 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.video;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+
+import androidx.media3.common.VideoSize;
+import androidx.media3.exoplayer.ExoPlaybackException;
+import androidx.media3.test.utils.FakeClock;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+/** Tests for {@link VideoFrameRenderControl}. */
+@RunWith(AndroidJUnit4.class)
+public class VideoFrameRenderControlTest {
+
+ private static final int VIDEO_WIDTH = 640;
+ private static final int VIDEO_HEIGHT = 480;
+
+ @Test
+ public void isReady_afterInstantiation_returnsFalse() {
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(
+ mock(VideoFrameRenderControl.FrameRenderer.class), createVideoFrameReleaseControl());
+
+ assertThat(videoFrameRenderControl.isReady()).isFalse();
+ }
+
+ @Test
+ public void releaseFirstFrame() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ assertThat(videoFrameRenderControl.isReady()).isTrue();
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(0L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+ }
+
+ @Test
+ public void releaseFirstAndSecondFrame() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ videoFrameReleaseControl.setClock(clock);
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameReleaseControl.onStarted();
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
+
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+ assertThat(videoFrameRenderControl.isReady()).isTrue();
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ // First frame.
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(0L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+ inOrder.verifyNoMoreInteractions();
+
+ // 5 seconds pass
+ clock.advanceTime(/* timeDiffMs= */ 5);
+ videoFrameRenderControl.render(/* positionUs= */ 5_000, /* elapsedRealtimeUs= */ 5_000);
+
+ // Second frame
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(10_000L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(false));
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ public void renderFrames_withStreamOffsetSetChange_firstFrameAgain() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ videoFrameReleaseControl.setClock(clock);
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameReleaseControl.onStarted();
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onStreamOffsetChange(
+ /* presentationTimeUs= */ 0, /* streamOffsetUs= */ 10_000);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ assertThat(videoFrameRenderControl.isReady()).isTrue();
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ // First frame has the first stream offset.
+ inOrder.verify(frameRenderer).renderFrame(anyLong(), eq(0L), eq(10_000L), eq(true));
+ inOrder.verifyNoMoreInteractions();
+
+ // 10 milliseconds pass
+ clock.advanceTime(/* timeDiffMs= */ 10);
+ videoFrameRenderControl.onStreamOffsetChange(
+ /* presentationTimeUs= */ 10_000, /* streamOffsetUs= */ 20_000);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
+ videoFrameRenderControl.render(/* positionUs= */ 10_000, /* elapsedRealtimeUs= */ 0);
+
+ // Second frame has the second stream offset and it is also a first frame.
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(10_000L),
+ /* streamOffsetUs= */ eq(20_000L),
+ /* isFirstFrame= */ eq(true));
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ public void dropFrames() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ FakeClock clock = new FakeClock(/* isAutoAdvancing= */ false);
+ VideoFrameReleaseControl videoFrameReleaseControl =
+ createVideoFrameReleaseControl(
+ new TestFrameTimingEvaluator(
+ /* shouldForceReleaseFrames= */ false,
+ /* shouldDropFrames= */ true,
+ /* shouldIgnoreFrames= */ false));
+ videoFrameReleaseControl.setClock(clock);
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameReleaseControl.onStarted();
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ // First frame was rendered because the fist frame is force released.
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(0L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+ inOrder.verifyNoMoreInteractions();
+
+ clock.advanceTime(/* timeDiffMs= */ 100);
+ videoFrameRenderControl.render(/* positionUs= */ 100_000, /* elapsedRealtimeUs= */ 100_000);
+
+ // Second frame was dropped.
+ inOrder.verify(frameRenderer).dropFrame();
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ public void flush_removesAvailableFramesForRendering_doesNotFlushOnVideoSizeChange()
+ throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameReleaseControl.onStarted();
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.flush();
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder.verifyNoMoreInteractions();
+
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 10_000);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ // First frame was rendered with pending video size change.
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(10_000L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+ inOrder.verifyNoMoreInteractions();
+ }
+
+ @Test
+ public void hasReleasedFrame_noFrameReleased_returnsFalse() {
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(
+ mock(VideoFrameRenderControl.FrameRenderer.class), videoFrameReleaseControl);
+
+ assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse();
+ }
+
+ @Test
+ public void hasReleasedFrame_frameIsReleased_returnsTrue() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(0L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+ assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isTrue();
+ }
+
+ @Test
+ public void hasReleasedFrame_frameIsReleasedAndFlushed_returnsFalse() throws Exception {
+ VideoFrameRenderControl.FrameRenderer frameRenderer =
+ mock(VideoFrameRenderControl.FrameRenderer.class);
+ VideoFrameReleaseControl videoFrameReleaseControl = createVideoFrameReleaseControl();
+ VideoFrameRenderControl videoFrameRenderControl =
+ new VideoFrameRenderControl(frameRenderer, videoFrameReleaseControl);
+
+ videoFrameReleaseControl.onEnabled(/* releaseFirstFrameBeforeStarted= */ true);
+ videoFrameRenderControl.onOutputSizeChanged(
+ /* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT);
+ videoFrameRenderControl.onOutputFrameAvailableForRendering(/* presentationTimeUs= */ 0);
+ videoFrameRenderControl.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
+
+ InOrder inOrder = Mockito.inOrder(frameRenderer);
+ inOrder
+ .verify(frameRenderer)
+ .onVideoSizeChanged(new VideoSize(/* width= */ VIDEO_WIDTH, /* height= */ VIDEO_HEIGHT));
+ inOrder
+ .verify(frameRenderer)
+ .renderFrame(
+ /* renderTimeNs= */ anyLong(),
+ /* presentationTimeUs= */ eq(0L),
+ /* streamOffsetUs= */ eq(0L),
+ /* isFirstFrame= */ eq(true));
+
+ videoFrameRenderControl.flush();
+
+ assertThat(videoFrameRenderControl.hasReleasedFrame(/* presentationTimeUs= */ 0)).isFalse();
+ }
+
+ private static VideoFrameReleaseControl createVideoFrameReleaseControl() {
+ return createVideoFrameReleaseControl(
+ new TestFrameTimingEvaluator(
+ /* shouldForceReleaseFrames= */ false,
+ /* shouldDropFrames= */ false,
+ /* shouldIgnoreFrames= */ false));
+ }
+
+ private static VideoFrameReleaseControl createVideoFrameReleaseControl(
+ VideoFrameReleaseControl.FrameTimingEvaluator frameTimingEvaluator) {
+ return new VideoFrameReleaseControl(
+ ApplicationProvider.getApplicationContext(),
+ frameTimingEvaluator,
+ /* allowedJoiningTimeMs= */ 0);
+ }
+
+ private static class TestFrameTimingEvaluator
+ implements VideoFrameReleaseControl.FrameTimingEvaluator {
+ private final boolean shouldForceReleaseFrames;
+ private final boolean shouldDropFrames;
+ private final boolean shouldIgnoreFrames;
+
+ public TestFrameTimingEvaluator(
+ boolean shouldForceReleaseFrames, boolean shouldDropFrames, boolean shouldIgnoreFrames) {
+ this.shouldForceReleaseFrames = shouldForceReleaseFrames;
+ this.shouldDropFrames = shouldDropFrames;
+ this.shouldIgnoreFrames = shouldIgnoreFrames;
+ }
+
+ @Override
+ public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
+ return shouldForceReleaseFrames;
+ }
+
+ @Override
+ public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
+ return shouldDropFrames;
+ }
+
+ @Override
+ public boolean shouldIgnoreFrame(
+ long earlyUs,
+ long positionUs,
+ long elapsedRealtimeUs,
+ boolean isLastFrame,
+ boolean treatDroppedBuffersAsSkipped)
+ throws ExoPlaybackException {
+ return shouldIgnoreFrames;
+ }
+ }
+}