diff --git a/Source/Core/VideoCommon/CPMemory.cpp b/Source/Core/VideoCommon/CPMemory.cpp
index 4781595e8c..0df0d9f1d8 100644
--- a/Source/Core/VideoCommon/CPMemory.cpp
+++ b/Source/Core/VideoCommon/CPMemory.cpp
@@ -9,6 +9,7 @@
 #include "Common/Logging/Log.h"
 #include "Core/DolphinAnalytics.h"
 #include "VideoCommon/CommandProcessor.h"
+#include "VideoCommon/VertexLoaderManager.h"
 
 // CP state
 CPState g_main_cp_state;
@@ -28,7 +29,7 @@ void DoCPState(PointerWrap& p)
   if (p.mode == PointerWrap::MODE_READ)
   {
     CopyPreprocessCPStateFromMain();
-    g_main_cp_state.bases_dirty = true;
+    VertexLoaderManager::g_bases_dirty = true;
   }
 }
 
@@ -154,8 +155,6 @@ void CPState::LoadCPReg(u8 sub_cmd, u32 value)
     }
 
     vtx_desc.low.Hex = value;
-    attr_dirty = BitSet32::AllTrue(CP_NUM_VAT_REG);
-    bases_dirty = true;
     break;
 
   case VCD_HI:
@@ -169,8 +168,6 @@ void CPState::LoadCPReg(u8 sub_cmd, u32 value)
     }
 
     vtx_desc.high.Hex = value;
-    attr_dirty = BitSet32::AllTrue(CP_NUM_VAT_REG);
-    bases_dirty = true;
     break;
 
   case CP_VAT_REG_A:
@@ -180,7 +177,6 @@ void CPState::LoadCPReg(u8 sub_cmd, u32 value)
       WARN_LOG_FMT(VIDEO, "CP_VAT_REG_A: Invalid VAT {}", sub_cmd - CP_VAT_REG_A);
     }
     vtx_attr[sub_cmd & CP_VAT_MASK].g0.Hex = value;
-    attr_dirty[sub_cmd & CP_VAT_MASK] = true;
     break;
 
   case CP_VAT_REG_B:
@@ -190,7 +186,6 @@ void CPState::LoadCPReg(u8 sub_cmd, u32 value)
       WARN_LOG_FMT(VIDEO, "CP_VAT_REG_B: Invalid VAT {}", sub_cmd - CP_VAT_REG_B);
     }
     vtx_attr[sub_cmd & CP_VAT_MASK].g1.Hex = value;
-    attr_dirty[sub_cmd & CP_VAT_MASK] = true;
     break;
 
   case CP_VAT_REG_C:
@@ -200,14 +195,12 @@ void CPState::LoadCPReg(u8 sub_cmd, u32 value)
       WARN_LOG_FMT(VIDEO, "CP_VAT_REG_C: Invalid VAT {}", sub_cmd - CP_VAT_REG_C);
     }
     vtx_attr[sub_cmd & CP_VAT_MASK].g2.Hex = value;
-    attr_dirty[sub_cmd & CP_VAT_MASK] = true;
     break;
 
   // Pointers to vertex arrays in GC RAM
   case ARRAY_BASE:
     array_bases[static_cast<CPArray>(sub_cmd & CP_ARRAY_MASK)] =
         value & CommandProcessor::GetPhysicalAddressMask();
-    bases_dirty = true;
     break;
 
   case ARRAY_STRIDE:
diff --git a/Source/Core/VideoCommon/CPMemory.h b/Source/Core/VideoCommon/CPMemory.h
index defac506b2..52b7038e7c 100644
--- a/Source/Core/VideoCommon/CPMemory.h
+++ b/Source/Core/VideoCommon/CPMemory.h
@@ -646,12 +646,6 @@ struct CPState final
   TVtxDesc vtx_desc;
   // Most games only use the first VtxAttr and simply reconfigure it all the time as needed.
   std::array<VAT, CP_NUM_VAT_REG> vtx_attr{};
-
-  // Attributes that actually belong to VertexLoaderManager:
-  BitSet32 attr_dirty{};
-  bool bases_dirty = false;
-  VertexLoaderBase* vertex_loaders[CP_NUM_VAT_REG]{};
-  int last_id = 0;
 };
 static_assert(std::is_trivially_copyable_v<CPState>);
 
diff --git a/Source/Core/VideoCommon/OpcodeDecoding.cpp b/Source/Core/VideoCommon/OpcodeDecoding.cpp
index 26e0f2da93..cefc88239d 100644
--- a/Source/Core/VideoCommon/OpcodeDecoding.cpp
+++ b/Source/Core/VideoCommon/OpcodeDecoding.cpp
@@ -58,17 +58,42 @@ public:
   OPCODE_CALLBACK(void OnCP(u8 command, u32 value))
   {
     m_cycles += 12;
+    const u8 sub_command = command & CP_COMMAND_MASK;
     if constexpr (!is_preprocess)
     {
-      // TODO: Move all dirty state checking here or to VertexLoaderManager,
-      // instead of it being in CPState
-      if (command == MATINDEX_A)
+      if (sub_command == MATINDEX_A)
         VertexShaderManager::SetTexMatrixChangedA(value);
-      else if (command == MATINDEX_B)
+      else if (sub_command == MATINDEX_B)
         VertexShaderManager::SetTexMatrixChangedB(value);
+      else if (sub_command == VCD_LO || sub_command == VCD_HI)
+      {
+        VertexLoaderManager::g_main_vat_dirty = BitSet8::AllTrue(CP_NUM_VAT_REG);
+        VertexLoaderManager::g_bases_dirty = true;
+      }
+      else if (sub_command == CP_VAT_REG_A || sub_command == CP_VAT_REG_B ||
+               sub_command == CP_VAT_REG_C)
+      {
+        VertexLoaderManager::g_main_vat_dirty[command & CP_VAT_MASK] = true;
+      }
+      else if (sub_command == ARRAY_BASE)
+      {
+        VertexLoaderManager::g_bases_dirty = true;
+      }
 
       INCSTAT(g_stats.this_frame.num_cp_loads);
     }
+    else if constexpr (is_preprocess)
+    {
+      if (sub_command == VCD_LO || sub_command == VCD_HI)
+      {
+        VertexLoaderManager::g_preprocess_vat_dirty = BitSet8::AllTrue(CP_NUM_VAT_REG);
+      }
+      else if (sub_command == CP_VAT_REG_A || sub_command == CP_VAT_REG_B ||
+               sub_command == CP_VAT_REG_C)
+      {
+        VertexLoaderManager::g_preprocess_vat_dirty[command & CP_VAT_MASK] = true;
+      }
+    }
     GetCPState().LoadCPReg(command, value);
   }
   OPCODE_CALLBACK(void OnBP(u8 command, u32 value))
diff --git a/Source/Core/VideoCommon/VertexLoaderARM64.cpp b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
index 75afefed16..106f1cfd9e 100644
--- a/Source/Core/VideoCommon/VertexLoaderARM64.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderARM64.cpp
@@ -406,7 +406,7 @@ void VertexLoaderARM64::GenerateVertexLoader()
   MOV(saved_count, count_reg);
 
   MOVP2R(stride_reg, g_main_cp_state.array_strides.data());
-  MOVP2R(arraybase_reg, VertexLoaderManager::cached_arraybases);
+  MOVP2R(arraybase_reg, VertexLoaderManager::cached_arraybases.data());
 
   if (need_scale)
     MOVP2R(scale_reg, scale_factors);
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.cpp b/Source/Core/VideoCommon/VertexLoaderManager.cpp
index 901e14044f..ef1e2f7aee 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderManager.cpp
@@ -58,9 +58,9 @@ std::array<VertexLoaderBase*, CP_NUM_VAT_REG> g_preprocess_vertex_loaders;
 void Init()
 {
   MarkAllDirty();
-  for (auto& map_entry : g_main_cp_state.vertex_loaders)
+  for (auto& map_entry : g_main_vertex_loaders)
     map_entry = nullptr;
-  for (auto& map_entry : g_preprocess_cp_state.vertex_loaders)
+  for (auto& map_entry : g_preprocess_vertex_loaders)
     map_entry = nullptr;
   SETSTAT(g_stats.num_vertex_loaders, 0);
 }
@@ -75,7 +75,7 @@ void Clear()
 void UpdateVertexArrayPointers()
 {
   // Anything to update?
-  if (!g_main_cp_state.bases_dirty)
+  if (!g_bases_dirty)
     return;
 
   // Some games such as Burnout 2 can put invalid addresses into
@@ -106,7 +106,7 @@ void UpdateVertexArrayPointers()
           Memory::GetPointer(g_main_cp_state.array_bases[CPArray::TexCoord0 + i]);
   }
 
-  g_main_cp_state.bases_dirty = false;
+  g_bases_dirty = false;
 }
 
 namespace
@@ -121,8 +121,8 @@ struct entry
 
 void MarkAllDirty()
 {
-  g_main_cp_state.attr_dirty = BitSet32::AllTrue(8);
-  g_preprocess_cp_state.attr_dirty = BitSet32::AllTrue(8);
+  g_main_vat_dirty = BitSet8::AllTrue(8);
+  g_preprocess_vat_dirty = BitSet8::AllTrue(8);
 }
 
 NativeVertexFormat* GetOrCreateMatchingFormat(const PortableVertexDeclaration& decl)
@@ -197,10 +197,12 @@ NativeVertexFormat* GetUberVertexFormat(const PortableVertexDeclaration& decl)
 static VertexLoaderBase* RefreshLoader(int vtx_attr_group, bool preprocess = false)
 {
   CPState* state = preprocess ? &g_preprocess_cp_state : &g_main_cp_state;
-  state->last_id = vtx_attr_group;
+  BitSet8& attr_dirty = preprocess ? g_preprocess_vat_dirty : g_main_vat_dirty;
+  auto& vertex_loaders = preprocess ? g_main_vertex_loaders : g_preprocess_vertex_loaders;
+  g_current_vat = vtx_attr_group;
 
   VertexLoaderBase* loader;
-  if (state->attr_dirty[vtx_attr_group])
+  if (attr_dirty[vtx_attr_group])
   {
     // We are not allowed to create a native vertex format on preprocessing as this is on the wrong
     // thread
@@ -230,12 +232,12 @@ static VertexLoaderBase* RefreshLoader(int vtx_attr_group, bool preprocess = fal
         native = g_renderer->CreateNativeVertexFormat(format);
       loader->m_native_vertex_format = native.get();
     }
-    state->vertex_loaders[vtx_attr_group] = loader;
-    state->attr_dirty[vtx_attr_group] = false;
+    vertex_loaders[vtx_attr_group] = loader;
+    attr_dirty[vtx_attr_group] = false;
   }
   else
   {
-    loader = state->vertex_loaders[vtx_attr_group];
+    loader = vertex_loaders[vtx_attr_group];
   }
 
   // Lookup pointers for any vertex arrays.
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.h b/Source/Core/VideoCommon/VertexLoaderManager.h
index 4eeae12742..d6bda13c00 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.h
+++ b/Source/Core/VideoCommon/VertexLoaderManager.h
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include <array>
 #include <memory>
 #include <string>
 #include <unordered_map>
@@ -57,4 +58,11 @@ extern u32 position_matrix_index[4];
 
 // VB_HAS_X. Bitmask telling what vertex components are present.
 extern u32 g_current_components;
+
+extern BitSet8 g_main_vat_dirty;
+extern BitSet8 g_preprocess_vat_dirty;
+extern bool g_bases_dirty;  // Main only
+extern u8 g_current_vat;    // Main only
+extern std::array<VertexLoaderBase*, CP_NUM_VAT_REG> g_main_vertex_loaders;
+extern std::array<VertexLoaderBase*, CP_NUM_VAT_REG> g_preprocess_vertex_loaders;
 }  // namespace VertexLoaderManager
diff --git a/Source/Core/VideoCommon/VertexShaderManager.cpp b/Source/Core/VideoCommon/VertexShaderManager.cpp
index 1b0bf55de7..25c7f79b30 100644
--- a/Source/Core/VideoCommon/VertexShaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexShaderManager.cpp
@@ -23,6 +23,7 @@
 #include "VideoCommon/FreeLookCamera.h"
 #include "VideoCommon/RenderBase.h"
 #include "VideoCommon/Statistics.h"
+#include "VideoCommon/VertexLoaderManager.h"
 #include "VideoCommon/VertexManagerBase.h"
 #include "VideoCommon/VideoCommon.h"
 #include "VideoCommon/VideoConfig.h"