diff --git a/Source/Core/VideoCommon/VertexManagerBase.cpp b/Source/Core/VideoCommon/VertexManagerBase.cpp
index 72f02927b5..670fcaf607 100644
--- a/Source/Core/VideoCommon/VertexManagerBase.cpp
+++ b/Source/Core/VideoCommon/VertexManagerBase.cpp
@@ -757,6 +757,16 @@ void VertexManagerBase::OnDraw()
 {
   m_draw_counter++;
 
+  // If the last efb copy was too close to the one before it, don't forget about it until the next
+  // efb copy happens (which might not be for a long time)
+  u32 diff = m_draw_counter - m_last_efb_copy_draw_counter;
+  if (m_unflushed_efb_copy && diff > MINIMUM_DRAW_CALLS_PER_COMMAND_BUFFER_FOR_READBACK)
+  {
+    g_renderer->Flush();
+    m_unflushed_efb_copy = false;
+    m_last_efb_copy_draw_counter = m_draw_counter;
+  }
+
   // If we didn't have any CPU access last frame, do nothing.
   if (m_scheduled_command_buffer_kicks.empty() || !m_allow_background_execution)
     return;
@@ -768,6 +778,8 @@ void VertexManagerBase::OnDraw()
   {
     // Kick a command buffer on the background thread.
     g_renderer->Flush();
+    m_unflushed_efb_copy = false;
+    m_last_efb_copy_draw_counter = m_draw_counter;
   }
 }
 
@@ -794,8 +806,12 @@ void VertexManagerBase::OnEFBCopyToRAM()
   const u32 diff = m_draw_counter - m_last_efb_copy_draw_counter;
   m_last_efb_copy_draw_counter = m_draw_counter;
   if (diff < MINIMUM_DRAW_CALLS_PER_COMMAND_BUFFER_FOR_READBACK)
+  {
+    m_unflushed_efb_copy = true;
     return;
+  }
 
+  m_unflushed_efb_copy = false;
   g_renderer->Flush();
 }
 
diff --git a/Source/Core/VideoCommon/VertexManagerBase.h b/Source/Core/VideoCommon/VertexManagerBase.h
index e4d029e9c7..ee1ce9408a 100644
--- a/Source/Core/VideoCommon/VertexManagerBase.h
+++ b/Source/Core/VideoCommon/VertexManagerBase.h
@@ -210,6 +210,7 @@ private:
   // CPU access tracking
   u32 m_draw_counter = 0;
   u32 m_last_efb_copy_draw_counter = 0;
+  bool m_unflushed_efb_copy = false;
   std::vector<u32> m_cpu_accesses_this_frame;
   std::vector<u32> m_scheduled_command_buffer_kicks;
   bool m_allow_background_execution = true;