fix(ffmpeg): apply upstream patches to fix VAAPI leaks and crashes (#369

)

.github/workflows/build-ffmpeg.yml

@@ -311,6 +311,7 @@ jobs:
           git apply -v --ignore-whitespace ../../ffmpeg_patches/ffmpeg/04-mfenc-lowlatency.patch
           git apply -v --ignore-whitespace ../../ffmpeg_patches/ffmpeg/05-vaapi-customized-surface-alignment.patch
           git apply -v --ignore-whitespace ../../ffmpeg_patches/ffmpeg/06-amfenc-query-timeout.patch
+          git apply -v --ignore-whitespace ../../ffmpeg_patches/ffmpeg/07-vaapi-leak.patch
 
       - name: Setup cross compilation
         id: cross

ffmpeg_patches/ffmpeg/07-vaapi-leak.patch

@@ -0,0 +1,66 @@
+From 48a1a12968345bf673db1e1cbb5c64bd3529c50c Mon Sep 17 00:00:00 2001
+From: David Rosca <[email protected]>
+Date: Tue, 15 Oct 2024 16:49:41 +0200
+Subject: [PATCH] hw_base_encode: Free pictures on close
+
+Fixes leaking recon surfaces with VAAPI.
+---
+ libavcodec/hw_base_encode.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/libavcodec/hw_base_encode.c b/libavcodec/hw_base_encode.c
+index 7b6ec97d3b..912c707a68 100644
+--- a/libavcodec/hw_base_encode.c
++++ b/libavcodec/hw_base_encode.c
+@@ -804,6 +804,11 @@ int ff_hw_base_encode_init(AVCodecContext *avctx, FFHWBaseEncodeContext *ctx)
+
+ int ff_hw_base_encode_close(FFHWBaseEncodeContext *ctx)
+ {
++    FFHWBaseEncodePicture *pic;
++
++    for (pic = ctx->pic_start; pic; pic = pic->next)
++        base_encode_pic_free(pic);
++
+     av_fifo_freep2(&ctx->encode_fifo);
+
+     av_frame_free(&ctx->frame);
+-- 
+2.25.1
+
+From c98810ab47fa1cf339b16045e27fbe12b3a19951 Mon Sep 17 00:00:00 2001
+From: Marvin Scholz <[email protected]>
+Date: Thu, 17 Oct 2024 20:23:40 +0200
+Subject: [PATCH] avcodec/hw_base_encode: fix use after free on close
+
+The way the linked list of images was freed caused a
+use after free, by accessing pic->next after pic was
+already freed.
+
+Regression from 48a1a12968345bf673db1e1cbb5c64bd3529c50c
+
+Fix CID1633236
+---
+ libavcodec/hw_base_encode.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/libavcodec/hw_base_encode.c b/libavcodec/hw_base_encode.c
+index 912c707a68..4d8bf4fe71 100644
+--- a/libavcodec/hw_base_encode.c
++++ b/libavcodec/hw_base_encode.c
+@@ -804,10 +804,10 @@ int ff_hw_base_encode_init(AVCodecContext *avctx, FFHWBaseEncodeContext *ctx)
+
+ int ff_hw_base_encode_close(FFHWBaseEncodeContext *ctx)
+ {
+-    FFHWBaseEncodePicture *pic;
+-
+-    for (pic = ctx->pic_start; pic; pic = pic->next)
++    for (FFHWBaseEncodePicture *pic = ctx->pic_start, *next_pic = pic; pic; pic = next_pic) {
++        next_pic = pic->next;
+         base_encode_pic_free(pic);
++    }
+
+     av_fifo_freep2(&ctx->encode_fifo);
+
+-- 
+2.25.1
+