diff --git a/Source/Core/Core/HW/Memmap.cpp b/Source/Core/Core/HW/Memmap.cpp
index 2800285c80..3adebdc700 100644
--- a/Source/Core/Core/HW/Memmap.cpp
+++ b/Source/Core/Core/HW/Memmap.cpp
@@ -216,9 +216,6 @@ void Init()
     }
   }
 
-  // TODO: Move this to the Jit
-  InitFastmemArena();
-
   if (wii)
     mmio_mapping = InitMMIOWii();
   else
diff --git a/Source/Core/Core/PowerPC/Jit64/Jit.cpp b/Source/Core/Core/PowerPC/Jit64/Jit.cpp
index db5c2711a9..19fcaed69e 100644
--- a/Source/Core/Core/PowerPC/Jit64/Jit.cpp
+++ b/Source/Core/Core/PowerPC/Jit64/Jit.cpp
@@ -333,6 +333,7 @@ void Jit64::Init()
   InitializeInstructionTables();
   EnableBlockLink();
 
+  jo.fastmem_arena = SConfig::GetInstance().bFastmem && Memory::InitFastmemArena();
   jo.optimizeGatherPipe = true;
   jo.accurateSinglePrecision = true;
   UpdateMemoryOptions();
@@ -393,6 +394,8 @@ void Jit64::Shutdown()
   FreeStack();
   FreeCodeSpace();
 
+  Memory::ShutdownFastmemArena();
+
   blocks.Shutdown();
   m_far_code.Shutdown();
   m_const_pool.Shutdown();
diff --git a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
index a60035d47a..a0b6fe9aae 100644
--- a/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
+++ b/Source/Core/Core/PowerPC/JitArm64/Jit.cpp
@@ -49,6 +49,8 @@ void JitArm64::Init()
   size_t child_code_size = SConfig::GetInstance().bMMU ? FARCODE_SIZE_MMU : FARCODE_SIZE;
   AllocCodeSpace(CODE_SIZE + child_code_size);
   AddChildCodeSpace(&farcode, child_code_size);
+
+  jo.fastmem_arena = SConfig::GetInstance().bFastmem && Memory::InitFastmemArena();
   jo.enableBlocklink = true;
   jo.optimizeGatherPipe = true;
   UpdateMemoryOptions();
@@ -133,6 +135,7 @@ void JitArm64::ClearCache()
 
 void JitArm64::Shutdown()
 {
+  Memory::ShutdownFastmemArena();
   FreeCodeSpace();
   blocks.Shutdown();
   FreeStack();
diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp b/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp
index b5ef8c2dc0..eeabe7737d 100644
--- a/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp
+++ b/Source/Core/Core/PowerPC/JitCommon/JitBase.cpp
@@ -45,6 +45,6 @@ bool JitBase::CanMergeNextInstructions(int count) const
 void JitBase::UpdateMemoryOptions()
 {
   bool any_watchpoints = PowerPC::memchecks.HasAny();
-  jo.fastmem = SConfig::GetInstance().bFastmem && (MSR.DR || !any_watchpoints);
+  jo.fastmem = SConfig::GetInstance().bFastmem && jo.fastmem_arena && (MSR.DR || !any_watchpoints);
   jo.memcheck = SConfig::GetInstance().bMMU || any_watchpoints;
 }
diff --git a/Source/Core/Core/PowerPC/JitCommon/JitBase.h b/Source/Core/Core/PowerPC/JitCommon/JitBase.h
index 6597384813..0d45293883 100644
--- a/Source/Core/Core/PowerPC/JitCommon/JitBase.h
+++ b/Source/Core/Core/PowerPC/JitCommon/JitBase.h
@@ -48,6 +48,7 @@ protected:
     bool optimizeGatherPipe;
     bool accurateSinglePrecision;
     bool fastmem;
+    bool fastmem_arena;
     bool memcheck;
     bool profile_blocks;
   };