feat(capture/windows): hook APIs to avoid output reparenting that bre…

…aks DDA (LizardByte#3530)

* Revert "feat(ddprobe): allow to manually specify gpu preference (LizardByte#3521)"

This reverts commit 6a233cb.

* Keep display revert delay input type change from 6a233cb

* Remove ddprobe

* feat(capture/windows): hook APIs to avoid output reparenting that breaks DDA

.codeql-prebuild-cpp-Windows.sh

@@ -5,26 +5,23 @@ set -e
 pacman --noconfirm -Syu
 
 # install dependencies
-pacman -S --noconfirm \
-  base-devel \
-  diffutils \
-  gcc \
-  git \
-  make \
-  mingw-w64-ucrt-x86_64-boost \
-  mingw-w64-ucrt-x86_64-cmake \
-  mingw-w64-ucrt-x86_64-cppwinrt \
-  mingw-w64-ucrt-x86_64-curl-winssl \
-  mingw-w64-ucrt-x86_64-graphviz \
-  mingw-w64-ucrt-x86_64-miniupnpc \
-  mingw-w64-ucrt-x86_64-nlohmann-json \
-  mingw-w64-ucrt-x86_64-nodejs \
-  mingw-w64-ucrt-x86_64-nsis \
-  mingw-w64-ucrt-x86_64-onevpl \
-  mingw-w64-ucrt-x86_64-openssl \
-  mingw-w64-ucrt-x86_64-opus \
-  mingw-w64-ucrt-x86_64-rust \
-  mingw-w64-ucrt-x86_64-toolchain
+dependencies=(
+  "git"
+  "mingw-w64-ucrt-x86_64-boost"
+  "mingw-w64-ucrt-x86_64-cmake"
+  "mingw-w64-ucrt-x86_64-cppwinrt"
+  "mingw-w64-ucrt-x86_64-curl-winssl"
+  "mingw-w64-ucrt-x86_64-MinHook"
+  "mingw-w64-ucrt-x86_64-miniupnpc"
+  "mingw-w64-ucrt-x86_64-nlohmann-json"
+  "mingw-w64-ucrt-x86_64-nodejs"
+  "mingw-w64-ucrt-x86_64-nsis"
+  "mingw-w64-ucrt-x86_64-onevpl"
+  "mingw-w64-ucrt-x86_64-openssl"
+  "mingw-w64-ucrt-x86_64-opus"
+  "mingw-w64-ucrt-x86_64-toolchain"
+)
+pacman -S --noconfirm "${dependencies[@]}"
 
 # build
 mkdir -p build

cmake/dependencies/common.cmake

@@ -25,7 +25,7 @@ include_directories(SYSTEM ${MINIUPNP_INCLUDE_DIRS})
 # ffmpeg pre-compiled binaries
 if(NOT DEFINED FFMPEG_PREPARED_BINARIES)
     if(WIN32)
-        set(FFMPEG_PLATFORM_LIBRARIES mfplat ole32 strmiids mfuuid vpl)
+        set(FFMPEG_PLATFORM_LIBRARIES mfplat ole32 strmiids mfuuid vpl MinHook)
     elseif(UNIX AND NOT APPLE)
         set(FFMPEG_PLATFORM_LIBRARIES numa va va-drm va-x11 X11)
     endif()

cmake/packaging/windows.cmake

@@ -11,7 +11,6 @@ install(TARGETS dxgi-info RUNTIME DESTINATION "tools" COMPONENT dxgi)
 install(TARGETS audio-info RUNTIME DESTINATION "tools" COMPONENT audio)
 
 # Mandatory tools
-install(TARGETS ddprobe RUNTIME DESTINATION "tools" COMPONENT application)
 install(TARGETS sunshinesvc RUNTIME DESTINATION "tools" COMPONENT application)
 
 # Mandatory scripts

docs/building.md

@@ -88,6 +88,7 @@ dependencies=(
   "mingw-w64-ucrt-x86_64-cppwinrt"
   "mingw-w64-ucrt-x86_64-curl-winssl"
   "mingw-w64-ucrt-x86_64-graphviz"  # Optional, for docs
+  "mingw-w64-ucrt-x86_64-MinHook"
   "mingw-w64-ucrt-x86_64-miniupnpc"
   "mingw-w64-ucrt-x86_64-nlohmann-json"
   "mingw-w64-ucrt-x86_64-nodejs"

docs/configuration.md

@@ -819,7 +819,7 @@ editing the `conf` file in a text editor. Use the examples as reference.
     </tr>
 </table>
 
-### [output_name](https://localhost:47990/config/#output_name)
+### output_name
 
 <table>
     <tr>

src/platform/windows/display_base.cpp

@@ -9,10 +9,22 @@
 #include <boost/algorithm/string/join.hpp>
 #include <boost/process/v1.hpp>
 
+#include <MinHook.h>
+
 // We have to include boost/process/v1.hpp before display.h due to WinSock.h,
 // but that prevents the definition of NTSTATUS so we must define it ourself.
 typedef long NTSTATUS;
 
+// Definition from the WDK's d3dkmthk.h
+typedef enum _D3DKMT_GPU_PREFERENCE_QUERY_STATE: DWORD {
+  D3DKMT_GPU_PREFERENCE_STATE_UNINITIALIZED,  ///< The GPU preference isn't initialized.
+  D3DKMT_GPU_PREFERENCE_STATE_HIGH_PERFORMANCE,  ///< The highest performing GPU is preferred.
+  D3DKMT_GPU_PREFERENCE_STATE_MINIMUM_POWER,  ///< The minimum-powered GPU is preferred.
+  D3DKMT_GPU_PREFERENCE_STATE_UNSPECIFIED,  ///< A GPU preference isn't specified.
+  D3DKMT_GPU_PREFERENCE_STATE_NOT_FOUND,  ///< A GPU preference isn't found.
+  D3DKMT_GPU_PREFERENCE_STATE_USER_SPECIFIED_GPU  ///< A specific GPU is preferred.
+} D3DKMT_GPU_PREFERENCE_QUERY_STATE;
+
 #include "display.h"
 #include "misc.h"
 #include "src/config.h"
@@ -525,6 +537,27 @@ namespace platf::dxgi {
     return false;
   }
 
+  /**
+   * @brief Hook for NtGdiDdDDIGetCachedHybridQueryValue() from win32u.dll.
+   * @param gpuPreference A pointer to the location where the preference will be written.
+   * @return Always STATUS_SUCCESS if valid arguments are provided.
+   */
+  NTSTATUS
+  __stdcall NtGdiDdDDIGetCachedHybridQueryValueHook(D3DKMT_GPU_PREFERENCE_QUERY_STATE *gpuPreference) {
+    // By faking a cached GPU preference state of D3DKMT_GPU_PREFERENCE_STATE_UNSPECIFIED, this will
+    // prevent DXGI from performing the normal GPU preference resolution that looks at the registry,
+    // power settings, and the hybrid adapter DDI interface to pick a GPU. Instead, we will not be
+    // bound to any specific GPU. This will prevent DXGI from performing output reparenting (moving
+    // outputs from their true location to the render GPU), which breaks DDA.
+    if (gpuPreference) {
+      *gpuPreference = D3DKMT_GPU_PREFERENCE_STATE_UNSPECIFIED;
+      return 0;  // STATUS_SUCCESS
+    }
+    else {
+      return STATUS_INVALID_PARAMETER;
+    }
+  }
+
   int
   display_base_t::init(const ::video::config_t &config, const std::string &display_name) {
     std::once_flag windows_cpp_once_flag;
@@ -534,13 +567,22 @@ namespace platf::dxgi {
 
       typedef BOOL (*User32_SetProcessDpiAwarenessContext)(DPI_AWARENESS_CONTEXT value);
 
-      auto user32 = LoadLibraryA("user32.dll");
-      auto f = (User32_SetProcessDpiAwarenessContext) GetProcAddress(user32, "SetProcessDpiAwarenessContext");
-      if (f) {
-        f(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);
+      {
+        auto user32 = LoadLibraryA("user32.dll");
+        auto f = (User32_SetProcessDpiAwarenessContext) GetProcAddress(user32, "SetProcessDpiAwarenessContext");
+        if (f) {
+          f(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);
+        }
+
+        FreeLibrary(user32);
       }
 
-      FreeLibrary(user32);
+      {
+        // We aren't calling MH_Uninitialize(), but that's okay because this hook lasts for the life of the process
+        MH_Initialize();
+        MH_CreateHookApi(L"win32u.dll", "NtGdiDdDDIGetCachedHybridQueryValue", (void *) NtGdiDdDDIGetCachedHybridQueryValueHook, nullptr);
+        MH_EnableHook(MH_ALL_HOOKS);
+      }
     });
 
     // Get rectangle of full desktop for absolute mouse coordinates
@@ -549,11 +591,6 @@ namespace platf::dxgi {
 
     HRESULT status;
 
-    // We must set the GPU preference before calling any DXGI APIs!
-    if (!probe_for_gpu_preference(display_name)) {
-      BOOST_LOG(warning) << "Failed to set GPU preference. Capture may not work!"sv;
-    }
-
     status = CreateDXGIFactory1(IID_IDXGIFactory1, (void **) &factory);
     if (FAILED(status)) {
       BOOST_LOG(error) << "Failed to create DXGIFactory1 [0x"sv << util::hex(status).to_string_view() << ']';