Merge pull request #7747 from stenzek/vulkan-shutdown

Vulkan: Shutdown fixes and cleanup/refactoring

Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp

@@ -21,17 +21,13 @@ CommandBufferManager::CommandBufferManager(bool use_threaded_submission)
 
 CommandBufferManager::~CommandBufferManager()
 {
-  // If the worker thread is enabled, wait for it to exit.
+  // If the worker thread is enabled, stop and block until it exits.
   if (m_use_threaded_submission)
   {
-    // Wait for all command buffers to be consumed by the worker thread.
-    m_submit_semaphore.Wait();
     m_submit_loop->Stop();
     m_submit_thread.join();
   }
 
-  vkDeviceWaitIdle(g_vulkan_context->GetDevice());
-
   DestroyCommandBuffers();
 }
 
@@ -121,6 +117,17 @@ void CommandBufferManager::DestroyCommandBuffers()
 
   for (FrameResources& resources : m_frame_resources)
   {
+    // The Vulkan spec section 5.2 says: "When a pool is destroyed, all command buffers allocated
+    // from the pool are freed.". So we don't need to free the command buffers, just the pools.
+    // We destroy the command pool first, to avoid any warnings from the validation layers about
+    // objects which are pending destruction being in-use.
+    if (resources.command_pool != VK_NULL_HANDLE)
+    {
+      vkDestroyCommandPool(device, resources.command_pool, nullptr);
+      resources.command_pool = VK_NULL_HANDLE;
+    }
+
+    // Destroy any pending objects.
     for (auto& it : resources.cleanup_resources)
       it();
     resources.cleanup_resources.clear();
@@ -130,24 +137,12 @@ void CommandBufferManager::DestroyCommandBuffers()
       vkDestroyFence(device, resources.fence, nullptr);
       resources.fence = VK_NULL_HANDLE;
     }
+
     if (resources.descriptor_pool != VK_NULL_HANDLE)
     {
       vkDestroyDescriptorPool(device, resources.descriptor_pool, nullptr);
       resources.descriptor_pool = VK_NULL_HANDLE;
     }
-    if (resources.command_buffers[0] != VK_NULL_HANDLE)
-    {
-      vkFreeCommandBuffers(device, resources.command_pool,
-                           static_cast<u32>(resources.command_buffers.size()),
-                           resources.command_buffers.data());
-
-      resources.command_buffers.fill(VK_NULL_HANDLE);
-    }
-    if (resources.command_pool != VK_NULL_HANDLE)
-    {
-      vkDestroyCommandPool(device, resources.command_pool, nullptr);
-      resources.command_pool = VK_NULL_HANDLE;
-    }
   }
 }
 
@@ -501,4 +496,4 @@ void CommandBufferManager::RemoveFencePointCallback(const void* key)
 }
 
 std::unique_ptr<CommandBufferManager> g_command_buffer_mgr;
-}
+}  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/Renderer.cpp

@@ -76,35 +76,6 @@ bool Renderer::Initialize()
 
   BindEFBToStateTracker();
 
-  if (!CreateSemaphores())
-  {
-    PanicAlert("Failed to create semaphores.");
-    return false;
-  }
-
-  if (!CompileShaders())
-  {
-    PanicAlert("Failed to compile shaders.");
-    return false;
-  }
-
-  // Swap chain render pass.
-  if (m_swap_chain)
-  {
-    m_swap_chain_render_pass =
-        g_object_cache->GetRenderPass(m_swap_chain->GetSurfaceFormat().format, VK_FORMAT_UNDEFINED,
-                                      1, VK_ATTACHMENT_LOAD_OP_LOAD);
-    m_swap_chain_clear_render_pass =
-        g_object_cache->GetRenderPass(m_swap_chain->GetSurfaceFormat().format, VK_FORMAT_UNDEFINED,
-                                      1, VK_ATTACHMENT_LOAD_OP_CLEAR);
-    if (m_swap_chain_render_pass == VK_NULL_HANDLE ||
-        m_swap_chain_clear_render_pass == VK_NULL_HANDLE)
-    {
-      PanicAlert("Failed to create swap chain render passes.");
-      return false;
-    }
-  }
-
   m_bounding_box = std::make_unique<BoundingBox>();
   if (!m_bounding_box->Initialize())
   {
@@ -140,51 +111,6 @@ bool Renderer::Initialize()
 void Renderer::Shutdown()
 {
   ::Renderer::Shutdown();
-
-  // Submit the current command buffer, in case there's a partial frame.
-  StateTracker::GetInstance()->EndRenderPass();
-  g_command_buffer_mgr->ExecuteCommandBuffer(false, true);
-
-  DestroyShaders();
-  DestroySemaphores();
-}
-
-bool Renderer::CreateSemaphores()
-{
-  // Create two semaphores, one that is triggered when the swapchain buffer is ready, another after
-  // submit and before present
-  VkSemaphoreCreateInfo semaphore_info = {
-      VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,  // VkStructureType          sType
-      nullptr,                                  // const void*              pNext
-      0                                         // VkSemaphoreCreateFlags   flags
-  };
-
-  VkResult res;
-  if ((res = vkCreateSemaphore(g_vulkan_context->GetDevice(), &semaphore_info, nullptr,
-                               &m_image_available_semaphore)) != VK_SUCCESS ||
-      (res = vkCreateSemaphore(g_vulkan_context->GetDevice(), &semaphore_info, nullptr,
-                               &m_rendering_finished_semaphore)) != VK_SUCCESS)
-  {
-    LOG_VULKAN_ERROR(res, "vkCreateSemaphore failed: ");
-    return false;
-  }
-
-  return true;
-}
-
-void Renderer::DestroySemaphores()
-{
-  if (m_image_available_semaphore)
-  {
-    vkDestroySemaphore(g_vulkan_context->GetDevice(), m_image_available_semaphore, nullptr);
-    m_image_available_semaphore = VK_NULL_HANDLE;
-  }
-
-  if (m_rendering_finished_semaphore)
-  {
-    vkDestroySemaphore(g_vulkan_context->GetDevice(), m_rendering_finished_semaphore, nullptr);
-    m_rendering_finished_semaphore = VK_NULL_HANDLE;
-  }
 }
 
 std::unique_ptr<AbstractTexture> Renderer::CreateTexture(const TextureConfig& config)
@@ -513,7 +439,8 @@ void Renderer::ClearScreen(const EFBRectangle& rc, bool color_enable, bool alpha
                          g_object_cache->GetPipelineLayout(PIPELINE_LAYOUT_STANDARD),
                          FramebufferManager::GetInstance()->GetEFBLoadRenderPass(),
                          g_shader_cache->GetPassthroughVertexShader(),
-                         g_shader_cache->GetPassthroughGeometryShader(), m_clear_fragment_shader);
+                         g_shader_cache->GetPassthroughGeometryShader(),
+                         g_shader_cache->GetClearFragmentShader());
 
   draw.SetMultisamplingState(FramebufferManager::GetInstance()->GetEFBMultisamplingState());
   draw.SetDepthState(depth_state);
@@ -557,7 +484,7 @@ void Renderer::BindBackbuffer(const ClearColor& clear_color)
   if (!g_command_buffer_mgr->CheckLastPresentFail())
   {
     // Grab the next image from the swap chain in preparation for drawing the window.
-    res = m_swap_chain->AcquireNextImage(m_image_available_semaphore);
+    res = m_swap_chain->AcquireNextImage();
   }
   else
   {
@@ -577,7 +504,7 @@ void Renderer::BindBackbuffer(const ClearColor& clear_color)
     m_swap_chain->ResizeSwapChain();
     BeginFrame();
     g_command_buffer_mgr->PrepareToSubmitCommandBuffer();
-    res = m_swap_chain->AcquireNextImage(m_image_available_semaphore);
+    res = m_swap_chain->AcquireNextImage();
   }
   if (res != VK_SUCCESS)
     PanicAlert("Failed to grab image from swap chain");
@@ -595,8 +522,8 @@ void Renderer::BindBackbuffer(const ClearColor& clear_color)
 
   // Draw to the backbuffer.
   VkRect2D region = {{0, 0}, {backbuffer->GetWidth(), backbuffer->GetHeight()}};
-  StateTracker::GetInstance()->SetRenderPass(m_swap_chain_render_pass,
+  StateTracker::GetInstance()->SetRenderPass(m_swap_chain->GetLoadRenderPass(),
-                                             m_swap_chain_clear_render_pass);
+                                             m_swap_chain->GetClearRenderPass());
   StateTracker::GetInstance()->SetFramebuffer(m_swap_chain->GetCurrentFramebuffer(), region);
 
   // Begin render pass for rendering to the swap chain.
@@ -612,18 +539,17 @@ void Renderer::PresentBackbuffer()
 
   // Transition the backbuffer to PRESENT_SRC to ensure all commands drawing
   // to it have finished before present.
-  Texture2D* backbuffer = m_swap_chain->GetCurrentTexture();
+  m_swap_chain->GetCurrentTexture()->TransitionToLayout(
-  backbuffer->TransitionToLayout(g_command_buffer_mgr->GetCurrentCommandBuffer(),
+      g_command_buffer_mgr->GetCurrentCommandBuffer(), VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
-                                 VK_IMAGE_LAYOUT_PRESENT_SRC_KHR);
 
   // Submit the current command buffer, signaling rendering finished semaphore when it's done
   // Because this final command buffer is rendering to the swap chain, we need to wait for
   // the available semaphore to be signaled before executing the buffer. This final submission
   // can happen off-thread in the background while we're preparing the next frame.
-  g_command_buffer_mgr->SubmitCommandBuffer(
+  g_command_buffer_mgr->SubmitCommandBuffer(true, m_swap_chain->GetImageAvailableSemaphore(),
-      true, m_image_available_semaphore, m_rendering_finished_semaphore,
+                                            m_swap_chain->GetRenderingFinishedSemaphore(),
-      m_swap_chain->GetSwapChain(), m_swap_chain->GetCurrentImageIndex());
+                                            m_swap_chain->GetSwapChain(),
-
+                                            m_swap_chain->GetCurrentImageIndex());
   BeginFrame();
 }
 
@@ -641,22 +567,22 @@ void Renderer::RenderXFBToScreen(const AbstractTexture* texture, const EFBRectan
 
     post_processor->BlitFromTexture(left_rect, rc,
                                     static_cast<const VKTexture*>(texture)->GetRawTexIdentifier(),
-                                    0, m_swap_chain_render_pass);
+                                    0, m_swap_chain->GetLoadRenderPass());
     post_processor->BlitFromTexture(right_rect, rc,
                                     static_cast<const VKTexture*>(texture)->GetRawTexIdentifier(),
-                                    1, m_swap_chain_render_pass);
+                                    1, m_swap_chain->GetLoadRenderPass());
   }
   else if (g_ActiveConfig.stereo_mode == StereoMode::QuadBuffer)
   {
     post_processor->BlitFromTexture(target_rc, rc,
                                     static_cast<const VKTexture*>(texture)->GetRawTexIdentifier(),
-                                    -1, m_swap_chain_render_pass);
+                                    -1, m_swap_chain->GetLoadRenderPass());
   }
   else
   {
     post_processor->BlitFromTexture(target_rc, rc,
                                     static_cast<const VKTexture*>(texture)->GetRawTexIdentifier(),
-                                    0, m_swap_chain_render_pass);
+                                    0, m_swap_chain->GetLoadRenderPass());
   }
 
   // The post-processor uses the old-style Vulkan draws, which mess with the tracked state.
@@ -725,7 +651,6 @@ void Renderer::OnConfigChanged(u32 bits)
   if (bits & (CONFIG_CHANGE_BIT_HOST_CONFIG | CONFIG_CHANGE_BIT_MULTISAMPLES))
   {
     RecreateEFBFramebuffer();
-    RecompileShaders();
     FramebufferManager::GetInstance()->RecompileShaders();
     g_shader_cache->ReloadPipelineCache();
     g_shader_cache->RecompileSharedShaders();
@@ -959,45 +884,4 @@ void Renderer::DrawIndexed(u32 base_index, u32 num_indices, u32 base_vertex)
   vkCmdDrawIndexed(g_command_buffer_mgr->GetCurrentCommandBuffer(), num_indices, 1, base_index,
                    base_vertex, 0);
 }
-
-void Renderer::RecompileShaders()
-{
-  DestroyShaders();
-  if (!CompileShaders())
-    PanicAlert("Failed to recompile shaders.");
-}
-
-bool Renderer::CompileShaders()
-{
-  static const char CLEAR_FRAGMENT_SHADER_SOURCE[] = R"(
-    layout(location = 0) in float3 uv0;
-    layout(location = 1) in float4 col0;
-    layout(location = 0) out float4 ocol0;
-
-    void main()
-    {
-      ocol0 = col0;
-    }
-
-  )";
-
-  std::string source = g_shader_cache->GetUtilityShaderHeader() + CLEAR_FRAGMENT_SHADER_SOURCE;
-  m_clear_fragment_shader = Util::CompileAndCreateFragmentShader(source);
-
-  return m_clear_fragment_shader != VK_NULL_HANDLE;
-}
-
-void Renderer::DestroyShaders()
-{
-  auto DestroyShader = [this](VkShaderModule& shader) {
-    if (shader != VK_NULL_HANDLE)
-    {
-      vkDestroyShaderModule(g_vulkan_context->GetDevice(), shader, nullptr);
-      shader = VK_NULL_HANDLE;
-    }
-  };
-
-  DestroyShader(m_clear_fragment_shader);
-}
-
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/Renderer.h

@@ -92,9 +92,6 @@ public:
   void PresentBackbuffer() override;
 
 private:
-  bool CreateSemaphores();
-  void DestroySemaphores();
-
   void BeginFrame();
 
   void CheckForSurfaceChange();
@@ -107,22 +104,10 @@ private:
   void RecreateEFBFramebuffer();
   void BindFramebuffer(const VKFramebuffer* fb);
 
-  void RecompileShaders();
-  bool CompileShaders();
-  void DestroyShaders();
-
-  VkSemaphore m_image_available_semaphore = VK_NULL_HANDLE;
-  VkSemaphore m_rendering_finished_semaphore = VK_NULL_HANDLE;
-  VkRenderPass m_swap_chain_render_pass = VK_NULL_HANDLE;
-  VkRenderPass m_swap_chain_clear_render_pass = VK_NULL_HANDLE;
-
   std::unique_ptr<SwapChain> m_swap_chain;
   std::unique_ptr<BoundingBox> m_bounding_box;
 
   // Keep a copy of sampler states to avoid cache lookups every draw
   std::array<SamplerState, NUM_PIXEL_SHADER_SAMPLERS> m_sampler_states = {};
-
-  // Shaders used for clear/blit.
-  VkShaderModule m_clear_fragment_shader = VK_NULL_HANDLE;
 };
 }

Source/Core/VideoBackends/Vulkan/ShaderCache.cpp

@@ -795,7 +795,19 @@ bool ShaderCache::CompileSharedShaders()
     }
   )";
 
-  std::string header = GetUtilityShaderHeader();
+  static const char CLEAR_FRAGMENT_SHADER_SOURCE[] = R"(
+    layout(location = 0) in float3 uv0;
+    layout(location = 1) in float4 col0;
+    layout(location = 0) out float4 ocol0;
+
+    void main()
+    {
+      ocol0 = col0;
+    }
+
+  )";
+
+  const std::string header = GetUtilityShaderHeader();
 
   m_screen_quad_vertex_shader =
       Util::CompileAndCreateVertexShader(header + SCREEN_QUAD_VERTEX_SHADER_SOURCE);
@@ -820,6 +832,11 @@ bool ShaderCache::CompileSharedShaders()
     }
   }
 
+  m_clear_fragment_shader =
+      Util::CompileAndCreateFragmentShader(header + CLEAR_FRAGMENT_SHADER_SOURCE);
+  if (m_clear_fragment_shader == VK_NULL_HANDLE)
+    return false;
+
   return true;
 }
 
@@ -837,5 +854,6 @@ void ShaderCache::DestroySharedShaders()
   DestroyShader(m_passthrough_vertex_shader);
   DestroyShader(m_screen_quad_geometry_shader);
   DestroyShader(m_passthrough_geometry_shader);
+  DestroyShader(m_clear_fragment_shader);
 }
-}
+}  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/ShaderCache.h

@@ -116,6 +116,7 @@ public:
   VkShaderModule GetPassthroughVertexShader() const { return m_passthrough_vertex_shader; }
   VkShaderModule GetScreenQuadGeometryShader() const { return m_screen_quad_geometry_shader; }
   VkShaderModule GetPassthroughGeometryShader() const { return m_passthrough_geometry_shader; }
+  VkShaderModule GetClearFragmentShader() const { return m_clear_fragment_shader; }
 
 private:
   bool CreatePipelineCache();
@@ -136,6 +137,7 @@ private:
   VkShaderModule m_passthrough_vertex_shader = VK_NULL_HANDLE;
   VkShaderModule m_screen_quad_geometry_shader = VK_NULL_HANDLE;
   VkShaderModule m_passthrough_geometry_shader = VK_NULL_HANDLE;
+  VkShaderModule m_clear_fragment_shader = VK_NULL_HANDLE;
 };
 
 extern std::unique_ptr<ShaderCache> g_shader_cache;

Source/Core/VideoBackends/Vulkan/SwapChain.cpp

@@ -36,6 +36,7 @@ SwapChain::~SwapChain()
   DestroySwapChainImages();
   DestroySwapChain();
   DestroySurface();
+  DestroySemaphores();
 }
 
 VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, void* display_handle, void* hwnd)
@@ -142,12 +143,53 @@ std::unique_ptr<SwapChain> SwapChain::Create(void* display_handle, void* native_
   std::unique_ptr<SwapChain> swap_chain =
       std::make_unique<SwapChain>(display_handle, native_handle, surface, vsync);
 
-  if (!swap_chain->CreateSwapChain() || !swap_chain->SetupSwapChainImages())
+  if (!swap_chain->CreateSemaphores() || !swap_chain->CreateSwapChain() ||
+      !swap_chain->SetupSwapChainImages())
+  {
     return nullptr;
+  }
 
   return swap_chain;
 }
 
+bool SwapChain::CreateSemaphores()
+{
+  // Create two semaphores, one that is triggered when the swapchain buffer is ready, another after
+  // submit and before present
+  VkSemaphoreCreateInfo semaphore_info = {
+      VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,  // VkStructureType          sType
+      nullptr,                                  // const void*              pNext
+      0                                         // VkSemaphoreCreateFlags   flags
+  };
+
+  VkResult res;
+  if ((res = vkCreateSemaphore(g_vulkan_context->GetDevice(), &semaphore_info, nullptr,
+                               &m_image_available_semaphore)) != VK_SUCCESS ||
+      (res = vkCreateSemaphore(g_vulkan_context->GetDevice(), &semaphore_info, nullptr,
+                               &m_rendering_finished_semaphore)) != VK_SUCCESS)
+  {
+    LOG_VULKAN_ERROR(res, "vkCreateSemaphore failed: ");
+    return false;
+  }
+
+  return true;
+}
+
+void SwapChain::DestroySemaphores()
+{
+  if (m_image_available_semaphore)
+  {
+    vkDestroySemaphore(g_vulkan_context->GetDevice(), m_image_available_semaphore, nullptr);
+    m_image_available_semaphore = VK_NULL_HANDLE;
+  }
+
+  if (m_rendering_finished_semaphore)
+  {
+    vkDestroySemaphore(g_vulkan_context->GetDevice(), m_rendering_finished_semaphore, nullptr);
+    m_rendering_finished_semaphore = VK_NULL_HANDLE;
+  }
+}
+
 bool SwapChain::SelectSurfaceFormat()
 {
   u32 format_count;
@@ -370,8 +412,15 @@ bool SwapChain::SetupSwapChainImages()
                                 images.data());
   ASSERT(res == VK_SUCCESS);
 
-  VkRenderPass render_pass = g_object_cache->GetRenderPass(
+  m_render_pass = g_object_cache->GetRenderPass(m_surface_format.format, VK_FORMAT_UNDEFINED, 1,
-      m_surface_format.format, VK_FORMAT_UNDEFINED, 1, VK_ATTACHMENT_LOAD_OP_CLEAR);
+                                                VK_ATTACHMENT_LOAD_OP_LOAD);
+  m_clear_render_pass = g_object_cache->GetRenderPass(m_surface_format.format, VK_FORMAT_UNDEFINED,
+                                                      1, VK_ATTACHMENT_LOAD_OP_CLEAR);
+  if (m_render_pass == VK_NULL_HANDLE || m_clear_render_pass == VK_NULL_HANDLE)
+  {
+    PanicAlert("Failed to get swap chain render passes.");
+    return false;
+  }
 
   m_swap_chain_images.reserve(image_count);
   for (uint32_t i = 0; i < image_count; i++)
@@ -388,7 +437,7 @@ bool SwapChain::SetupSwapChainImages()
     VkFramebufferCreateInfo framebuffer_info = {VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
                                                 nullptr,
                                                 0,
-                                                render_pass,
+                                                m_render_pass,
                                                 1,
                                                 &view,
                                                 m_width,
@@ -428,11 +477,11 @@ void SwapChain::DestroySwapChain()
   m_swap_chain = VK_NULL_HANDLE;
 }
 
-VkResult SwapChain::AcquireNextImage(VkSemaphore available_semaphore)
+VkResult SwapChain::AcquireNextImage()
 {
-  VkResult res =
+  VkResult res = vkAcquireNextImageKHR(g_vulkan_context->GetDevice(), m_swap_chain, UINT64_MAX,
-      vkAcquireNextImageKHR(g_vulkan_context->GetDevice(), m_swap_chain, UINT64_MAX,
+                                       m_image_available_semaphore, VK_NULL_HANDLE,
-                            available_semaphore, VK_NULL_HANDLE, &m_current_swap_chain_image_index);
+                                       &m_current_swap_chain_image_index);
   if (res != VK_SUCCESS && res != VK_ERROR_OUT_OF_DATE_KHR && res != VK_SUBOPTIMAL_KHR)
     LOG_VULKAN_ERROR(res, "vkAcquireNextImageKHR failed: ");
 

Source/Core/VideoBackends/Vulkan/SwapChain.h

@@ -53,8 +53,12 @@ public:
   {
     return m_swap_chain_images[m_current_swap_chain_image_index].framebuffer;
   }
+  VkRenderPass GetLoadRenderPass() const { return m_render_pass; }
+  VkRenderPass GetClearRenderPass() const { return m_clear_render_pass; }
+  VkSemaphore GetImageAvailableSemaphore() const { return m_image_available_semaphore; }
+  VkSemaphore GetRenderingFinishedSemaphore() const { return m_rendering_finished_semaphore; }
 
-  VkResult AcquireNextImage(VkSemaphore available_semaphore);
+  VkResult AcquireNextImage();
 
   bool RecreateSurface(void* native_handle);
   bool ResizeSwapChain();
@@ -64,6 +68,9 @@ public:
   bool SetVSync(bool enabled);
 
 private:
+  bool CreateSemaphores();
+  void DestroySemaphores();
+
   bool SelectSurfaceFormat();
   bool SelectPresentMode();
 
@@ -94,6 +101,12 @@ private:
   std::vector<SwapChainImage> m_swap_chain_images;
   u32 m_current_swap_chain_image_index = 0;
 
+  VkSemaphore m_image_available_semaphore = VK_NULL_HANDLE;
+  VkSemaphore m_rendering_finished_semaphore = VK_NULL_HANDLE;
+
+  VkRenderPass m_render_pass = VK_NULL_HANDLE;
+  VkRenderPass m_clear_render_pass = VK_NULL_HANDLE;
+
   u32 m_width = 0;
   u32 m_height = 0;
   u32 m_layers = 0;

Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl

@@ -29,130 +29,9 @@ VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceProperties, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceQueueFamilyProperties, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceMemoryProperties, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkCreateDevice, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyDevice, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkEnumerateDeviceExtensionProperties, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkEnumerateDeviceLayerProperties, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetDeviceQueue, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkQueueSubmit, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkQueueWaitIdle, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDeviceWaitIdle, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkAllocateMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkFreeMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkMapMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkUnmapMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkFlushMappedMemoryRanges, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkInvalidateMappedMemoryRanges, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetDeviceMemoryCommitment, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkBindBufferMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkBindImageMemory, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetBufferMemoryRequirements, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetImageMemoryRequirements, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetImageSparseMemoryRequirements, true)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceSparseImageFormatProperties, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkQueueBindSparse, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateFence, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyFence, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkResetFences, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetFenceStatus, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkWaitForFences, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateSemaphore, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroySemaphore, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetEventStatus, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkSetEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkResetEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateQueryPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyQueryPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetQueryPoolResults, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateBufferView, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyBufferView, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetImageSubresourceLayout, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateImageView, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyImageView, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateShaderModule, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyShaderModule, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreatePipelineCache, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyPipelineCache, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetPipelineCacheData, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkMergePipelineCaches, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateGraphicsPipelines, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateComputePipelines, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyPipeline, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreatePipelineLayout, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyPipelineLayout, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateSampler, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroySampler, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateDescriptorSetLayout, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyDescriptorSetLayout, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateDescriptorPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyDescriptorPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkResetDescriptorPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkAllocateDescriptorSets, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkFreeDescriptorSets, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkUpdateDescriptorSets, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateFramebuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyFramebuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateRenderPass, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyRenderPass, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkGetRenderAreaGranularity, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCreateCommandPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkDestroyCommandPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkResetCommandPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkAllocateCommandBuffers, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkFreeCommandBuffers, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkBeginCommandBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkEndCommandBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkResetCommandBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBindPipeline, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetViewport, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetScissor, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetLineWidth, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetDepthBias, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetBlendConstants, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetDepthBounds, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetStencilCompareMask, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetStencilWriteMask, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetStencilReference, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBindDescriptorSets, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBindIndexBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBindVertexBuffers, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDraw, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDrawIndexed, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDrawIndirect, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDrawIndexedIndirect, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDispatch, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdDispatchIndirect, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdCopyBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdCopyImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBlitImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdCopyBufferToImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdCopyImageToBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdUpdateBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdFillBuffer, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdClearColorImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdClearDepthStencilImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdClearAttachments, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdResolveImage, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdSetEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdResetEvent, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdWaitEvents, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdPipelineBarrier, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBeginQuery, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdEndQuery, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdResetQueryPool, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdWriteTimestamp, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdCopyQueryPoolResults, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdPushConstants, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdBeginRenderPass, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdNextSubpass, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdEndRenderPass, true)
-VULKAN_INSTANCE_ENTRY_POINT(vkCmdExecuteCommands, true)
-
 VULKAN_INSTANCE_ENTRY_POINT(vkDestroySurfaceKHR, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceSurfaceSupportKHR, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceSurfaceCapabilitiesKHR, false)
@@ -192,6 +71,126 @@ VULKAN_INSTANCE_ENTRY_POINT(vkDebugReportMessageEXT, false)
 
 #ifdef VULKAN_DEVICE_ENTRY_POINT
 
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyDevice, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetDeviceQueue, true)
+VULKAN_DEVICE_ENTRY_POINT(vkQueueSubmit, true)
+VULKAN_DEVICE_ENTRY_POINT(vkQueueWaitIdle, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDeviceWaitIdle, true)
+VULKAN_DEVICE_ENTRY_POINT(vkAllocateMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkFreeMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkMapMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkUnmapMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkFlushMappedMemoryRanges, true)
+VULKAN_DEVICE_ENTRY_POINT(vkInvalidateMappedMemoryRanges, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetDeviceMemoryCommitment, true)
+VULKAN_DEVICE_ENTRY_POINT(vkBindBufferMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkBindImageMemory, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetBufferMemoryRequirements, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetImageMemoryRequirements, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetImageSparseMemoryRequirements, true)
+VULKAN_DEVICE_ENTRY_POINT(vkQueueBindSparse, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateFence, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyFence, true)
+VULKAN_DEVICE_ENTRY_POINT(vkResetFences, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetFenceStatus, true)
+VULKAN_DEVICE_ENTRY_POINT(vkWaitForFences, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateSemaphore, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroySemaphore, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetEventStatus, true)
+VULKAN_DEVICE_ENTRY_POINT(vkSetEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkResetEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateQueryPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyQueryPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetQueryPoolResults, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateBufferView, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyBufferView, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetImageSubresourceLayout, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateImageView, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyImageView, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateShaderModule, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyShaderModule, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreatePipelineCache, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyPipelineCache, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetPipelineCacheData, true)
+VULKAN_DEVICE_ENTRY_POINT(vkMergePipelineCaches, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateGraphicsPipelines, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateComputePipelines, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyPipeline, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreatePipelineLayout, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyPipelineLayout, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateSampler, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroySampler, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateDescriptorSetLayout, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyDescriptorSetLayout, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateDescriptorPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyDescriptorPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkResetDescriptorPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkAllocateDescriptorSets, true)
+VULKAN_DEVICE_ENTRY_POINT(vkFreeDescriptorSets, true)
+VULKAN_DEVICE_ENTRY_POINT(vkUpdateDescriptorSets, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateFramebuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyFramebuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateRenderPass, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyRenderPass, true)
+VULKAN_DEVICE_ENTRY_POINT(vkGetRenderAreaGranularity, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCreateCommandPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkDestroyCommandPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkResetCommandPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkAllocateCommandBuffers, true)
+VULKAN_DEVICE_ENTRY_POINT(vkFreeCommandBuffers, true)
+VULKAN_DEVICE_ENTRY_POINT(vkBeginCommandBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkEndCommandBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkResetCommandBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBindPipeline, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetViewport, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetScissor, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetLineWidth, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetDepthBias, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetBlendConstants, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetDepthBounds, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetStencilCompareMask, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetStencilWriteMask, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetStencilReference, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBindDescriptorSets, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBindIndexBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBindVertexBuffers, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDraw, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDrawIndexed, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDrawIndirect, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDrawIndexedIndirect, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDispatch, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdDispatchIndirect, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdCopyBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdCopyImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBlitImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdCopyBufferToImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdCopyImageToBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdUpdateBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdFillBuffer, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdClearColorImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdClearDepthStencilImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdClearAttachments, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdResolveImage, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdResetEvent, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdWaitEvents, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdPipelineBarrier, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBeginQuery, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdEndQuery, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdResetQueryPool, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdWriteTimestamp, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdCopyQueryPoolResults, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdPushConstants, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdBeginRenderPass, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdNextSubpass, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdEndRenderPass, true)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdExecuteCommands, true)
 VULKAN_DEVICE_ENTRY_POINT(vkCreateSwapchainKHR, false)
 VULKAN_DEVICE_ENTRY_POINT(vkDestroySwapchainKHR, false)
 VULKAN_DEVICE_ENTRY_POINT(vkGetSwapchainImagesKHR, false)