Refactor to use pixel format helpers in VAAPI

LizardByte · Jan 28, 2024 · 9a3553d · 9a3553d
1 parent 3b11bc8
commit 9a3553d
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 22 deletions.
diff --git a/src/platform/linux/graphics.cpp b/src/platform/linux/graphics.cpp
@@ -7,6 +7,10 @@
 
 #include <fcntl.h>
 
+extern "C" {
+#include <libavutil/pixdesc.h>
+}
+
 // I want to have as little build dependencies as possible
 // There aren't that many DRM_FORMAT I need to use, so define them here
 //
@@ -806,10 +810,36 @@ namespace egl {
   }
 
   std::optional<sws_t>
-  sws_t::make(int in_width, int in_height, int out_width, int out_height, GLint gl_tex_internal_fmt) {
+  sws_t::make(int in_width, int in_height, int out_width, int out_height, AVPixelFormat format) {
+    GLint gl_format;
+
+    // Decide the bit depth format of the backing texture based the target frame format
+    auto fmt_desc = av_pix_fmt_desc_get(format);
+    switch (fmt_desc->comp[0].depth) {
+      case 8:
+        gl_format = GL_RGBA8;
+        break;
+
+      case 10:
+        gl_format = GL_RGB10_A2;
+        break;
+
+      case 12:
+        gl_format = GL_RGBA12;
+        break;
+
+      case 16:
+        gl_format = GL_RGBA16;
+        break;
+
+      default:
+        BOOST_LOG(error) << "Unsupported pixel format for EGL frame: "sv << (int) format;
+        return std::nullopt;
+    }
+
     auto tex = gl::tex_t::make(2);
     gl::ctx.BindTexture(GL_TEXTURE_2D, tex[0]);
-    gl::ctx.TexStorage2D(GL_TEXTURE_2D, 1, gl_tex_internal_fmt, in_width, in_height);
+    gl::ctx.TexStorage2D(GL_TEXTURE_2D, 1, gl_format, in_width, in_height);
 
     return make(in_width, in_height, out_width, out_height, std::move(tex));
   }

diff --git a/src/platform/linux/graphics.h b/src/platform/linux/graphics.h
@@ -316,7 +316,7 @@ namespace egl {
     static std::optional<sws_t>
     make(int in_width, int in_height, int out_width, int out_height, gl::tex_t &&tex);
     static std::optional<sws_t>
-    make(int in_width, int in_height, int out_width, int out_height, GLint gl_tex_internal_fmt);
+    make(int in_width, int in_height, int out_width, int out_height, AVPixelFormat format);
 
     // Convert the loaded image into the first two framebuffers
     int

diff --git a/src/platform/linux/vaapi.cpp b/src/platform/linux/vaapi.cpp
@@ -195,25 +195,7 @@ namespace va {
         return -1;
       }
 
-      // Decide the bit depth format of the backing texture based the target frame format
-      GLint gl_format;
-      switch (hw_frames_ctx->sw_format) {
-        case AV_PIX_FMT_YUV420P:
-        case AV_PIX_FMT_NV12:
-          gl_format = GL_RGBA8;
-          break;
-
-        case AV_PIX_FMT_YUV420P10:
-        case AV_PIX_FMT_P010:
-          gl_format = GL_RGB10_A2;
-          break;
-
-        default:
-          BOOST_LOG(error) << "Unsupported pixel format for VA frame: "sv << hw_frames_ctx->sw_format;
-          return -1;
-      }
-
-      auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height, gl_format);
+      auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height, hw_frames_ctx->sw_format);
       if (!sws_opt) {
         return -1;
       }