Video: implement color correction to match the NTSC and PAL color spaces (and gamma) that GC and Wii targeted.

To further increase the accuracy of the post process phase, I've added (scRGB) HDR support, which is necessary
to fully display the PAL and NTSC-J color spaces, and also to improve the quality of post process texture samplings and
do them in linear space instead of gamma space (which is very important when playing at low resolutions).
For SDR, the quality is also slightly increased, at least if any post process runs, as the buffer is now
R10G10B10A2 (on Vulkan, DX11 and DX12) if supported; previously it was R8G8B8A8 but the alpha bits were wasted.

Gamma correction is arguably the most important thing as Dolphin on Windows outputted in "sRGB" (implicitly)
as that's what Windows expects by default, though sRGB gamma is very different from the gamma commonly used
by video standards dating to the pre HDR era (roughly gamma 2.35).

Additionally, the addition of HDR support (which is pretty straight forward and minimal), added support for
our own custom AutoHDR shaders, which would allow us to achieve decent looking HDR in Dolphin games without
having to use SpecialK or Windows 11 AutoHDR. Both of which don't necessarily play nice with older games
with strongly different and simpler lighting. HDR should also be supported in Linux.
Development of my own AutoHDR shader is almost complete and will come next.

This has been carefully tested and there should be no regression in any of the different features that Dolphin
offers, like multisampling, stereo rendering, other post processes, etc etc.

Fixes: https://bugs.dolphin-emu.org/issues/8941

Co-authored-by: EndlesslyFlowering <EndlesslyFlowering@protonmail.com>
Co-authored-by: Dogway <lin_ares@hotmail.com>

Source/Core/VideoBackends/D3D/D3DBase.cpp

@@ -15,6 +15,7 @@
 #include "VideoBackends/D3D/D3DState.h"
 #include "VideoBackends/D3D/DXTexture.h"
 #include "VideoBackends/D3DCommon/D3DCommon.h"
+#include "VideoCommon/FramebufferManager.h"
 #include "VideoCommon/VideoConfig.h"
 
 namespace DX11
@@ -202,12 +203,14 @@ std::vector<u32> GetAAModes(u32 adapter_index)
   if (temp_feature_level == D3D_FEATURE_LEVEL_10_0)
     return {};
 
+  const DXGI_FORMAT target_format =
+      D3DCommon::GetDXGIFormatForAbstractFormat(FramebufferManager::GetEFBColorFormat(), false);
   std::vector<u32> aa_modes;
   for (u32 samples = 1; samples <= D3D11_MAX_MULTISAMPLE_SAMPLE_COUNT; ++samples)
   {
     UINT quality_levels = 0;
-    if (SUCCEEDED(temp_device->CheckMultisampleQualityLevels(DXGI_FORMAT_R8G8B8A8_UNORM, samples,
-                                                             &quality_levels)) &&
+    if (SUCCEEDED(
+            temp_device->CheckMultisampleQualityLevels(target_format, samples, &quality_levels)) &&
         quality_levels > 0)
     {
       aa_modes.push_back(samples);

Source/Core/VideoBackends/D3D/D3DGfx.cpp

@@ -175,6 +175,9 @@ void Gfx::OnConfigChanged(u32 bits)
   // Quad-buffer changes require swap chain recreation.
   if (bits & CONFIG_CHANGE_BIT_STEREO_MODE && m_swap_chain)
     m_swap_chain->SetStereo(SwapChain::WantsStereo());
+
+  if (bits & CONFIG_CHANGE_BIT_HDR && m_swap_chain)
+    m_swap_chain->SetHDR(SwapChain::WantsHDR());
 }
 
 void Gfx::CheckForSwapChainChanges()

Source/Core/VideoBackends/D3D/D3DMain.cpp

@@ -114,6 +114,7 @@ void VideoBackend::FillBackendInfo()
   g_Config.backend_info.bSupportsSettingObjectNames = true;
   g_Config.backend_info.bSupportsPartialMultisampleResolve = true;
   g_Config.backend_info.bSupportsDynamicVertexLoader = false;
+  g_Config.backend_info.bSupportsHDROutput = true;
 
   g_Config.backend_info.Adapters = D3DCommon::GetAdapterNames();
   g_Config.backend_info.AAModes = D3D::GetAAModes(g_Config.iAdapter);

Source/Core/VideoBackends/D3D/D3DSwapChain.cpp

@@ -21,7 +21,7 @@ std::unique_ptr<SwapChain> SwapChain::Create(const WindowSystemInfo& wsi)
 {
   std::unique_ptr<SwapChain> swap_chain =
       std::make_unique<SwapChain>(wsi, D3D::dxgi_factory.Get(), D3D::device.Get());
-  if (!swap_chain->CreateSwapChain(WantsStereo()))
+  if (!swap_chain->CreateSwapChain(WantsStereo(), WantsHDR()))
     return nullptr;
 
   return swap_chain;

Source/Core/VideoBackends/D3D12/D3D12Gfx.cpp

@@ -421,6 +421,12 @@ void Gfx::OnConfigChanged(u32 bits)
     m_swap_chain->SetStereo(SwapChain::WantsStereo());
   }
 
+  if (m_swap_chain && bits & CONFIG_CHANGE_BIT_HDR)
+  {
+    ExecuteCommandList(true);
+    m_swap_chain->SetHDR(SwapChain::WantsHDR());
+  }
+
   // Wipe sampler cache if force texture filtering or anisotropy changes.
   if (bits & (CONFIG_CHANGE_BIT_ANISOTROPY | CONFIG_CHANGE_BIT_FORCE_TEXTURE_FILTERING))
   {

Source/Core/VideoBackends/D3D12/D3D12SwapChain.cpp

@@ -22,7 +22,7 @@ std::unique_ptr<SwapChain> SwapChain::Create(const WindowSystemInfo& wsi)
 {
   std::unique_ptr<SwapChain> swap_chain = std::make_unique<SwapChain>(
       wsi, g_dx_context->GetDXGIFactory(), g_dx_context->GetCommandQueue());
-  if (!swap_chain->CreateSwapChain(WantsStereo()))
+  if (!swap_chain->CreateSwapChain(WantsStereo(), WantsHDR()))
     return nullptr;
 
   return swap_chain;

Source/Core/VideoBackends/D3D12/DX12Context.cpp

@@ -16,6 +16,7 @@
 #include "VideoBackends/D3D12/Common.h"
 #include "VideoBackends/D3D12/D3D12StreamBuffer.h"
 #include "VideoBackends/D3D12/DescriptorHeapManager.h"
+#include "VideoCommon/FramebufferManager.h"
 #include "VideoCommon/VideoConfig.h"
 
 namespace DX12
@@ -65,11 +66,13 @@ std::vector<u32> DXContext::GetAAModes(u32 adapter_index)
       return {};
   }
 
+  const DXGI_FORMAT target_format =
+      D3DCommon::GetDXGIFormatForAbstractFormat(FramebufferManager::GetEFBColorFormat(), false);
   std::vector<u32> aa_modes;
   for (u32 samples = 1; samples < D3D12_MAX_MULTISAMPLE_SAMPLE_COUNT; ++samples)
   {
     D3D12_FEATURE_DATA_MULTISAMPLE_QUALITY_LEVELS multisample_quality_levels = {};
-    multisample_quality_levels.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+    multisample_quality_levels.Format = target_format;
     multisample_quality_levels.SampleCount = samples;
 
     temp_device->CheckFeatureSupport(D3D12_FEATURE_MULTISAMPLE_QUALITY_LEVELS,

Source/Core/VideoBackends/D3D12/VideoBackend.cpp

@@ -90,6 +90,7 @@ void VideoBackend::FillBackendInfo()
   g_Config.backend_info.bSupportsPartialMultisampleResolve = true;
   g_Config.backend_info.bSupportsDynamicVertexLoader = true;
   g_Config.backend_info.bSupportsVSLinePointExpand = true;
+  g_Config.backend_info.bSupportsHDROutput = true;
 
   // We can only check texture support once we have a device.
   if (g_dx_context)

Source/Core/VideoBackends/D3DCommon/SwapChain.cpp

@@ -51,13 +51,18 @@ bool SwapChain::WantsStereo()
   return g_ActiveConfig.stereo_mode == StereoMode::QuadBuffer;
 }
 
+bool SwapChain::WantsHDR()
+{
+  return g_ActiveConfig.bHDR;
+}
+
 u32 SwapChain::GetSwapChainFlags() const
 {
   // This flag is necessary if we want to use a flip-model swapchain without locking the framerate
   return m_allow_tearing_supported ? DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING : 0;
 }
 
-bool SwapChain::CreateSwapChain(bool stereo)
+bool SwapChain::CreateSwapChain(bool stereo, bool hdr)
 {
   RECT client_rc;
   if (GetClientRect(static_cast<HWND>(m_wsi.render_surface), &client_rc))
@@ -66,6 +71,9 @@ bool SwapChain::CreateSwapChain(bool stereo)
     m_height = client_rc.bottom - client_rc.top;
   }
 
+  m_stereo = false;
+  m_hdr = false;
+
   // Try using the Win8 version if available.
   Microsoft::WRL::ComPtr<IDXGIFactory2> dxgi_factory2;
   HRESULT hr = m_dxgi_factory.As(&dxgi_factory2);
@@ -81,6 +89,7 @@ bool SwapChain::CreateSwapChain(bool stereo)
     swap_chain_desc.SampleDesc.Count = 1;
     swap_chain_desc.SampleDesc.Quality = 0;
     swap_chain_desc.Format = GetDXGIFormatForAbstractFormat(m_texture_format, false);
+
     swap_chain_desc.Scaling = DXGI_SCALING_STRETCH;
     swap_chain_desc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD;
     swap_chain_desc.Stereo = stereo;
@@ -108,6 +117,8 @@ bool SwapChain::CreateSwapChain(bool stereo)
   // support the newer DXGI interface aren't going to support DX12 anyway.
   if (FAILED(hr))
   {
+    hdr = false;
+
     DXGI_SWAP_CHAIN_DESC desc = {};
     desc.BufferDesc.Width = m_width;
     desc.BufferDesc.Height = m_height;
@@ -138,6 +149,37 @@ bool SwapChain::CreateSwapChain(bool stereo)
     WARN_LOG_FMT(VIDEO, "MakeWindowAssociation() failed: {}", Common::HRWrap(hr));
 
   m_stereo = stereo;
+
+  if (hdr)
+  {
+    // Only try to activate HDR here, to avoid failing when creating the swapchain
+    // (we can't know if the format is supported upfront)
+    Microsoft::WRL::ComPtr<IDXGISwapChain4> swap_chain4;
+    hr = m_swap_chain->QueryInterface(IID_PPV_ARGS(&swap_chain4));
+    if (SUCCEEDED(hr))
+    {
+      UINT color_space_support = 0;
+      // Note that this should succeed even if HDR is not currently engaged on the monitor,
+      // but it should display fine nonetheless.
+      // We need to check for DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 as checking for
+      // scRGB always returns false (DX bug).
+      hr = swap_chain4->CheckColorSpaceSupport(DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020,
+                                               &color_space_support);
+      if (SUCCEEDED(hr) && (color_space_support & DXGI_SWAP_CHAIN_COLOR_SPACE_SUPPORT_FLAG_PRESENT))
+      {
+        hr = swap_chain4->ResizeBuffers(SWAP_CHAIN_BUFFER_COUNT, 0, 0,
+                                        GetDXGIFormatForAbstractFormat(m_texture_format_hdr, false),
+                                        GetSwapChainFlags());
+        if (SUCCEEDED(hr))
+        {
+          hr = swap_chain4->SetColorSpace1(DXGI_COLOR_SPACE_RGB_FULL_G10_NONE_P709);
+          if (SUCCEEDED(hr))
+            m_hdr = hdr;
+        }
+      }
+    }
+  }
+
   if (!CreateSwapChainBuffers())
   {
     PanicAlertFmt("Failed to create swap chain buffers");
@@ -164,12 +206,19 @@ bool SwapChain::ResizeSwapChain()
 {
   DestroySwapChainBuffers();
 
-  HRESULT hr = m_swap_chain->ResizeBuffers(SWAP_CHAIN_BUFFER_COUNT, 0, 0,
-                                           GetDXGIFormatForAbstractFormat(m_texture_format, false),
+  // The swap chain fills up the size of the window if no size is specified
+  HRESULT hr = m_swap_chain->ResizeBuffers(SWAP_CHAIN_BUFFER_COUNT, 0, 0, DXGI_FORMAT_UNKNOWN,
                                            GetSwapChainFlags());
+
   if (FAILED(hr))
     WARN_LOG_FMT(VIDEO, "ResizeBuffers() failed: {}", Common::HRWrap(hr));
 
+  Microsoft::WRL::ComPtr<IDXGISwapChain4> swap_chain4;
+  hr = m_swap_chain->QueryInterface(IID_PPV_ARGS(&swap_chain4));
+  if (SUCCEEDED(hr))
+    hr = swap_chain4->SetColorSpace1(m_hdr ? DXGI_COLOR_SPACE_RGB_FULL_G10_NONE_P709 :
+                                             DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709);
+
   DXGI_SWAP_CHAIN_DESC desc;
   if (SUCCEEDED(m_swap_chain->GetDesc(&desc)))
   {
@@ -186,10 +235,28 @@ void SwapChain::SetStereo(bool stereo)
     return;
 
   DestroySwapChain();
-  if (!CreateSwapChain(stereo))
+  // Do not try to re-activate HDR here if it had already failed
+  if (!CreateSwapChain(stereo, m_hdr))
   {
     PanicAlertFmt("Failed to switch swap chain stereo mode");
-    CreateSwapChain(false);
+    CreateSwapChain(false, false);
+  }
+}
+
+void SwapChain::SetHDR(bool hdr)
+{
+  if (m_hdr == hdr)
+    return;
+
+  // NOTE: as an optimization here we could just call "ResizeSwapChain()"
+  // by adding some code to check if we could change the format to HDR.
+
+  DestroySwapChain();
+  // Do not try to re-activate stereo mode here if it had already failed
+  if (!CreateSwapChain(m_stereo, hdr))
+  {
+    PanicAlertFmt("Failed to switch swap chain SDR/HDR mode");
+    CreateSwapChain(false, false);
   }
 }
 
@@ -249,7 +316,8 @@ bool SwapChain::ChangeSurface(void* native_handle)
 {
   DestroySwapChain();
   m_wsi.render_surface = native_handle;
-  return CreateSwapChain(m_stereo);
+  // We only keep the swap chain settings (HDR/Stereo) that had successfully applied beofre
+  return CreateSwapChain(m_stereo, m_hdr);
 }
 
 }  // namespace D3DCommon

Source/Core/VideoBackends/D3DCommon/SwapChain.h

@@ -25,8 +25,13 @@ public:
   // Returns true if the stereo mode is quad-buffering.
   static bool WantsStereo();
 
+  static bool WantsHDR();
+
   IDXGISwapChain* GetDXGISwapChain() const { return m_swap_chain.Get(); }
-  AbstractTextureFormat GetFormat() const { return m_texture_format; }
+  AbstractTextureFormat GetFormat() const
+  {
+    return m_hdr ? m_texture_format_hdr : m_texture_format;
+  }
   u32 GetWidth() const { return m_width; }
   u32 GetHeight() const { return m_height; }
 
@@ -43,10 +48,11 @@ public:
   bool ChangeSurface(void* native_handle);
   bool ResizeSwapChain();
   void SetStereo(bool stereo);
+  void SetHDR(bool hdr);
 
 protected:
   u32 GetSwapChainFlags() const;
-  bool CreateSwapChain(bool stereo);
+  bool CreateSwapChain(bool stereo = false, bool hdr = false);
   void DestroySwapChain();
 
   virtual bool CreateSwapChainBuffers() = 0;
@@ -56,12 +62,14 @@ protected:
   Microsoft::WRL::ComPtr<IDXGIFactory> m_dxgi_factory;
   Microsoft::WRL::ComPtr<IDXGISwapChain> m_swap_chain;
   Microsoft::WRL::ComPtr<IUnknown> m_d3d_device;
-  AbstractTextureFormat m_texture_format = AbstractTextureFormat::RGBA8;
+  const AbstractTextureFormat m_texture_format = AbstractTextureFormat::RGB10_A2;
+  const AbstractTextureFormat m_texture_format_hdr = AbstractTextureFormat::RGBA16F;
 
   u32 m_width = 1;
   u32 m_height = 1;
 
   bool m_stereo = false;
+  bool m_hdr = false;
   bool m_allow_tearing_supported = false;
   bool m_has_fullscreen = false;
   bool m_fullscreen_request = false;

Source/Core/VideoBackends/Vulkan/VKGfx.cpp

@@ -399,7 +399,7 @@ void VKGfx::OnConfigChanged(u32 bits)
   }
 
   // For quad-buffered stereo we need to change the layer count, so recreate the swap chain.
-  if (m_swap_chain && bits & CONFIG_CHANGE_BIT_STEREO_MODE)
+  if (m_swap_chain && (bits & CONFIG_CHANGE_BIT_STEREO_MODE) || (bits & CONFIG_CHANGE_BIT_HDR))
   {
     ExecuteCommandBuffer(false, true);
     m_swap_chain->RecreateSwapChain();

Source/Core/VideoBackends/Vulkan/VKSwapChain.cpp

@@ -154,7 +154,7 @@ bool SwapChain::SelectSurfaceFormat()
                                              &format_count, surface_formats.data());
   ASSERT(res == VK_SUCCESS);
 
-  // If there is a single undefined surface format, the device doesn't care, so we'll just use RGBA
+  // If there is a single undefined surface format, the device doesn't care, so we'll just use RGBA8
   if (surface_formats[0].format == VK_FORMAT_UNDEFINED)
   {
     m_surface_format.format = VK_FORMAT_R8G8B8A8_UNORM;
@@ -162,22 +162,61 @@ bool SwapChain::SelectSurfaceFormat()
     return true;
   }
 
-  // Try to find a suitable format.
+  const VkSurfaceFormatKHR* surface_format_RGBA8 = nullptr;
+  const VkSurfaceFormatKHR* surface_format_BGRA8 = nullptr;
+  const VkSurfaceFormatKHR* surface_format_RGB10_A2 = nullptr;
+  const VkSurfaceFormatKHR* surface_format_RGBA16F_scRGB = nullptr;
+
+  // Try to find all suitable formats.
   for (const VkSurfaceFormatKHR& surface_format : surface_formats)
   {
-    // Some drivers seem to return a SRGB format here (Intel Mesa).
-    // This results in gamma correction when presenting to the screen, which we don't want.
-    // Use a linear format instead, if this is the case.
+    // Some drivers seem to return a RGBA8 SRGB format here (Intel Mesa).
+    // Some other drivers return both a RGBA8 SRGB and UNORM formats (Nvidia).
+    // This results in gamma correction when presenting to the screen, which we don't want,
+    // because we already apply gamma ourselves, and we might not use sRGB gamma.
+    // Force using a linear format instead, if this is the case.
     VkFormat format = VKTexture::GetLinearFormat(surface_format.format);
+    if (format == VK_FORMAT_R8G8B8A8_UNORM)
+      surface_format_RGBA8 = &surface_format;
+    else if (format == VK_FORMAT_B8G8R8A8_UNORM)
+      surface_format_BGRA8 = &surface_format;
+    else if (format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 &&
+             surface_format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)
+      surface_format_RGB10_A2 = &surface_format;
+    else if (format == VK_FORMAT_R16G16B16A16_SFLOAT &&
+             surface_format.colorSpace == VK_COLOR_SPACE_EXTENDED_SRGB_LINEAR_EXT)
+      surface_format_RGBA16F_scRGB = &surface_format;
+    else
+      continue;
+  }
+
+  const VkSurfaceFormatKHR* surface_format = nullptr;
+
+  // Pick the best format.
+  // "g_ActiveConfig" might not have been been updated yet.
+  if (g_Config.bHDR && surface_format_RGBA16F_scRGB)
+    surface_format = surface_format_RGBA16F_scRGB;
+  else if (surface_format_RGB10_A2)
+    surface_format = surface_format_RGB10_A2;
+  else if (surface_format_RGBA8)
+    surface_format = surface_format_RGBA8;
+  else if (surface_format_BGRA8)
+    surface_format = surface_format_BGRA8;
+
+  if (surface_format)
+  {
+    const VkFormat format = VKTexture::GetLinearFormat(surface_format->format);
     if (format == VK_FORMAT_R8G8B8A8_UNORM)
       m_texture_format = AbstractTextureFormat::RGBA8;
     else if (format == VK_FORMAT_B8G8R8A8_UNORM)
       m_texture_format = AbstractTextureFormat::BGRA8;
-    else
-      continue;
+    else if (format == VK_FORMAT_A2B10G10R10_UNORM_PACK32)
+      m_texture_format = AbstractTextureFormat::RGB10_A2;
+    else if (format == VK_FORMAT_R16G16B16A16_SFLOAT)
+      m_texture_format = AbstractTextureFormat::RGBA16F;
 
     m_surface_format.format = format;
-    m_surface_format.colorSpace = surface_format.colorSpace;
+    m_surface_format.colorSpace = surface_format->colorSpace;
     return true;
   }
 

Source/Core/VideoBackends/Vulkan/VulkanContext.cpp

@@ -382,6 +382,7 @@ void VulkanContext::PopulateBackendInfo(VideoConfig* config)
   config->backend_info.bSupportsPartialMultisampleResolve = true;  // Assumed support.
   config->backend_info.bSupportsDynamicVertexLoader = true;        // Assumed support.
   config->backend_info.bSupportsVSLinePointExpand = true;          // Assumed support.
+  config->backend_info.bSupportsHDROutput = true;                  // Assumed support.
 }
 
 void VulkanContext::PopulateBackendInfoAdapters(VideoConfig* config, const GPUList& gpu_list)