Merge pull request #5589 from stenzek/vulkan-validation-fixes

Vulkan: Don't transition image layouts inside render passes

Source/Core/VideoBackends/Vulkan/Renderer.cpp

@@ -329,6 +329,10 @@ void Renderer::BeginFrame()
   // Activate a new command list, and restore state ready for the next draw
   g_command_buffer_mgr->ActivateCommandBuffer();
 
+  // Restore the EFB color texture to color attachment ready for rendering the next frame.
+  FramebufferManager::GetInstance()->GetEFBColorTexture()->TransitionToLayout(
+      g_command_buffer_mgr->GetCurrentCommandBuffer(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+
   // Ensure that the state tracker rebinds everything, and allocates a new set
   // of descriptors out of the next pool.
   StateTracker::GetInstance()->InvalidateDescriptorSets();
@@ -516,8 +520,7 @@ void Renderer::SwapImpl(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height
   // If MSAA is enabled, and we're not using XFB, we need to resolve the EFB framebuffer before
   // rendering the final image to the screen, or dumping the frame. This is because we can't resolve
   // an image within a render pass, which will have already started by the time it is used.
-  if (g_ActiveConfig.iMultisamples > 1 && !g_ActiveConfig.bUseXFB)
+  TransitionBuffersForSwap(scaled_efb_rect, xfb_sources, xfb_count);
-    ResolveEFBForSwap(scaled_efb_rect);
 
   // Render the frame dump image if enabled.
   if (IsFrameDumping())
@@ -588,16 +591,44 @@ void Renderer::SwapImpl(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height
   TextureCache::GetInstance()->Cleanup(frameCount);
 }
 
-void Renderer::ResolveEFBForSwap(const TargetRectangle& scaled_rect)
+void Renderer::TransitionBuffersForSwap(const TargetRectangle& scaled_rect,
+                                        const XFBSourceBase* const* xfb_sources, u32 xfb_count)
 {
-  // While the source rect can be out-of-range when drawing, the resolve rectangle must be within
+  VkCommandBuffer command_buffer = g_command_buffer_mgr->GetCurrentCommandBuffer();
-  // the bounds of the texture.
+
-  VkRect2D region = {
+  if (!g_ActiveConfig.bUseXFB)
-      {scaled_rect.left, scaled_rect.top},
+  {
-      {static_cast<u32>(scaled_rect.GetWidth()), static_cast<u32>(scaled_rect.GetHeight())}};
+    // Drawing EFB direct.
-  region = Util::ClampRect2D(region, FramebufferManager::GetInstance()->GetEFBWidth(),
+    if (g_ActiveConfig.iMultisamples > 1)
-                             FramebufferManager::GetInstance()->GetEFBHeight());
+    {
-  FramebufferManager::GetInstance()->ResolveEFBColorTexture(region);
+      // While the source rect can be out-of-range when drawing, the resolve rectangle must be
+      // within the bounds of the texture.
+      VkRect2D region = {
+          {scaled_rect.left, scaled_rect.top},
+          {static_cast<u32>(scaled_rect.GetWidth()), static_cast<u32>(scaled_rect.GetHeight())}};
+      region = Util::ClampRect2D(region, FramebufferManager::GetInstance()->GetEFBWidth(),
+                                 FramebufferManager::GetInstance()->GetEFBHeight());
+
+      Vulkan::Texture2D* rtex = FramebufferManager::GetInstance()->ResolveEFBColorTexture(region);
+      rtex->TransitionToLayout(command_buffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+    }
+    else
+    {
+      FramebufferManager::GetInstance()->GetEFBColorTexture()->TransitionToLayout(
+          command_buffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+    }
+
+    return;
+  }
+
+  // Drawing XFB sources, so transition all of them.
+  // Don't need the EFB, so leave it as-is.
+  for (u32 i = 0; i < xfb_count; i++)
+  {
+    const XFBSource* xfb_source = static_cast<const XFBSource*>(xfb_sources[i]);
+    xfb_source->GetTexture()->GetTexture()->TransitionToLayout(
+        command_buffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  }
 }
 
 void Renderer::DrawFrame(VkRenderPass render_pass, const TargetRectangle& target_rect,
@@ -622,18 +653,9 @@ void Renderer::DrawEFB(VkRenderPass render_pass, const TargetRectangle& target_r
       g_ActiveConfig.iMultisamples > 1 ?
           FramebufferManager::GetInstance()->GetResolvedEFBColorTexture() :
           FramebufferManager::GetInstance()->GetEFBColorTexture();
-  efb_color_texture->TransitionToLayout(g_command_buffer_mgr->GetCurrentCommandBuffer(),
-                                        VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
 
   // Copy EFB -> backbuffer
   BlitScreen(render_pass, target_rect, scaled_efb_rect, efb_color_texture);
-
-  // Restore the EFB color texture to color attachment ready for rendering the next frame.
-  if (efb_color_texture == FramebufferManager::GetInstance()->GetEFBColorTexture())
-  {
-    FramebufferManager::GetInstance()->GetEFBColorTexture()->TransitionToLayout(
-        g_command_buffer_mgr->GetCurrentCommandBuffer(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
-  }
 }
 
 void Renderer::DrawVirtualXFB(VkRenderPass render_pass, const TargetRectangle& target_rect,
@@ -643,9 +665,6 @@ void Renderer::DrawVirtualXFB(VkRenderPass render_pass, const TargetRectangle& t
   for (u32 i = 0; i < xfb_count; ++i)
   {
     const XFBSource* xfb_source = static_cast<const XFBSource*>(xfb_sources[i]);
-    xfb_source->GetTexture()->GetTexture()->TransitionToLayout(
-        g_command_buffer_mgr->GetCurrentCommandBuffer(), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
-
     TargetRectangle source_rect = xfb_source->sourceRc;
     TargetRectangle draw_rect;
 
@@ -679,9 +698,6 @@ void Renderer::DrawRealXFB(VkRenderPass render_pass, const TargetRectangle& targ
   for (u32 i = 0; i < xfb_count; ++i)
   {
     const XFBSource* xfb_source = static_cast<const XFBSource*>(xfb_sources[i]);
-    xfb_source->GetTexture()->GetTexture()->TransitionToLayout(
-        g_command_buffer_mgr->GetCurrentCommandBuffer(), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
-
     TargetRectangle source_rect = xfb_source->sourceRc;
     TargetRectangle draw_rect = target_rect;
     source_rect.right -= fb_stride - fb_width;

Source/Core/VideoBackends/Vulkan/Renderer.h

@@ -87,7 +87,10 @@ private:
   bool CompileShaders();
   void DestroyShaders();
 
-  void ResolveEFBForSwap(const TargetRectangle& scaled_rect);
+  // Transitions EFB/XFB buffers to SHADER_READ_ONLY, ready for presenting/dumping.
+  // If MSAA is enabled, and XFB is disabled, also resolves the EFB buffer.
+  void TransitionBuffersForSwap(const TargetRectangle& scaled_rect,
+                                const XFBSourceBase* const* xfb_sources, u32 xfb_count);
 
   // Draw either the EFB, or specified XFB sources to the currently-bound framebuffer.
   void DrawFrame(VkRenderPass render_pass, const TargetRectangle& target_rect,

Source/Core/VideoBackends/Vulkan/TextureCache.cpp

@@ -152,9 +152,19 @@ void TextureCache::DecodeTextureOnGPU(TCacheEntryBase* entry, u32 dst_level, con
                                       u32 aligned_width, u32 aligned_height, u32 row_stride,
                                       const u8* palette, TlutFormat palette_format)
 {
-  m_texture_converter->DecodeTexture(static_cast<TCacheEntry*>(entry), dst_level, data, data_size,
+  // Group compute shader dispatches together in the init command buffer. That way we don't have to
-                                     format, width, height, aligned_width, aligned_height,
+  // pay a penalty for switching from graphics->compute, or end/restart our render pass.
-                                     row_stride, palette, palette_format);
+  VkCommandBuffer command_buffer = g_command_buffer_mgr->GetCurrentInitCommandBuffer();
+  m_texture_converter->DecodeTexture(command_buffer, static_cast<TCacheEntry*>(entry), dst_level,
+                                     data, data_size, format, width, height, aligned_width,
+                                     aligned_height, row_stride, palette, palette_format);
+
+  // Last mip level? Ensure the texture is ready for use.
+  if (dst_level == (entry->config.levels - 1))
+  {
+    static_cast<TCacheEntry*>(entry)->GetTexture()->TransitionToLayout(
+        command_buffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  }
 }
 
 void TextureCache::CopyTextureRectangle(TCacheEntry* dst_texture,
@@ -207,7 +217,8 @@ void TextureCache::ScaleTextureRectangle(TCacheEntry* dst_texture,
   _assert_msg_(VIDEO, dst_texture->config.rendertarget,
                "Destination texture for partial copy is not a rendertarget");
 
-  // Render pass expects dst_texture to be in SHADER_READ_ONLY state.
+  // Render pass expects dst_texture to be in COLOR_ATTACHMENT_OPTIMAL state.
+  // src_texture should already be in SHADER_READ_ONLY state, but transition in case (XFB).
   src_texture->TransitionToLayout(g_command_buffer_mgr->GetCurrentCommandBuffer(),
                                   VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
   dst_texture->GetTexture()->TransitionToLayout(g_command_buffer_mgr->GetCurrentCommandBuffer(),
@@ -363,8 +374,9 @@ void TextureCache::TCacheEntry::Load(u32 level, u32 width, u32 height, u32 row_l
   // should insert an explicit pipeline barrier just in case (done by TransitionToLayout).
   //
   // We transition to TRANSFER_DST, ready for the image copy, and leave the texture in this state.
-  // This is so that the remaining mip levels can be uploaded without barriers, and then when the
+  // When the last mip level is uploaded, we transition to SHADER_READ_ONLY, ready for use. This is
-  // texture is used, it can be transitioned to SHADER_READ_ONLY (see TCacheEntry::Bind).
+  // because we can't transition in a render pass, and we don't necessarily know when this texture
+  // is going to be used.
   m_texture->TransitionToLayout(g_command_buffer_mgr->GetCurrentInitCommandBuffer(),
                                 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
 
@@ -430,6 +442,13 @@ void TextureCache::TCacheEntry::Load(u32 level, u32 width, u32 height, u32 row_l
   vkCmdCopyBufferToImage(g_command_buffer_mgr->GetCurrentInitCommandBuffer(), upload_buffer,
                          m_texture->GetImage(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1,
                          &image_copy);
+
+  // Last mip level? We shouldn't be doing any further uploads now, so transition for rendering.
+  if (level == (config.levels - 1))
+  {
+    m_texture->TransitionToLayout(g_command_buffer_mgr->GetCurrentInitCommandBuffer(),
+                                  VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  }
 }
 
 void TextureCache::TCacheEntry::FromRenderTarget(bool is_depth_copy, const EFBRectangle& src_rect,
@@ -497,7 +516,7 @@ void TextureCache::TCacheEntry::FromRenderTarget(bool is_depth_copy, const EFBRe
   // Transition the EFB back to its original layout.
   src_texture->TransitionToLayout(command_buffer, original_layout);
 
-  // Render pass transitions texture to SHADER_READ_ONLY.
+  // Ensure texture is in SHADER_READ_ONLY layout, ready for usage.
   m_texture->TransitionToLayout(command_buffer, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
 }
 
@@ -518,8 +537,9 @@ void TextureCache::TCacheEntry::CopyRectangleFromTexture(const TCacheEntryBase*
 
 void TextureCache::TCacheEntry::Bind(unsigned int stage)
 {
-  m_texture->TransitionToLayout(g_command_buffer_mgr->GetCurrentCommandBuffer(),
+  // Texture should always be in SHADER_READ_ONLY layout prior to use.
-                                VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  // This is so we don't need to transition during render passes.
+  _assert_(m_texture->GetLayout() == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
   StateTracker::GetInstance()->SetTexture(stage, m_texture->GetView());
 }
 

Source/Core/VideoBackends/Vulkan/TextureConverter.cpp

@@ -416,7 +416,8 @@ bool TextureConverter::SupportsTextureDecoding(TextureFormat format, TlutFormat
   return true;
 }
 
-void TextureConverter::DecodeTexture(TextureCache::TCacheEntry* entry, u32 dst_level,
+void TextureConverter::DecodeTexture(VkCommandBuffer command_buffer,
+                                     TextureCache::TCacheEntry* entry, u32 dst_level,
                                      const u8* data, size_t data_size, TextureFormat format,
                                      u32 width, u32 height, u32 aligned_width, u32 aligned_height,
                                      u32 row_stride, const u8* palette, TlutFormat palette_format)
@@ -498,11 +499,6 @@ void TextureConverter::DecodeTexture(TextureCache::TCacheEntry* entry, u32 dst_l
     break;
   }
 
-  // Place compute shader dispatches together in the init command buffer.
-  // That way we don't have to pay a penalty for switching from graphics->compute,
-  // or end/restart our render pass.
-  VkCommandBuffer command_buffer = g_command_buffer_mgr->GetCurrentInitCommandBuffer();
-
   // Dispatch compute to temporary texture.
   ComputeShaderDispatcher dispatcher(command_buffer,
                                      g_object_cache->GetPipelineLayout(PIPELINE_LAYOUT_COMPUTE),

Source/Core/VideoBackends/Vulkan/TextureConverter.h

@@ -49,10 +49,10 @@ public:
                                    u32 src_width, u32 src_stride, u32 src_height);
 
   bool SupportsTextureDecoding(TextureFormat format, TlutFormat palette_format);
-  void DecodeTexture(TextureCache::TCacheEntry* entry, u32 dst_level, const u8* data,
+  void DecodeTexture(VkCommandBuffer command_buffer, TextureCache::TCacheEntry* entry,
-                     size_t data_size, TextureFormat format, u32 width, u32 height,
+                     u32 dst_level, const u8* data, size_t data_size, TextureFormat format,
-                     u32 aligned_width, u32 aligned_height, u32 row_stride, const u8* palette,
+                     u32 width, u32 height, u32 aligned_width, u32 aligned_height, u32 row_stride,
-                     TlutFormat palette_format);
+                     const u8* palette, TlutFormat palette_format);
 
 private:
   static const u32 ENCODING_TEXTURE_WIDTH = EFB_WIDTH * 4;