Merge pull request #12336 from AdmiralCurtiss/lazymemory-ringbuffer

Common/MemArenaWin: Rewrite LazyMemoryRegion by repeatedly mapping the same block of memory for unwritten regions.

Source/Core/Common/MemArena.h

@@ -14,6 +14,15 @@ namespace Common
 {
 #ifdef _WIN32
 struct WindowsMemoryRegion;
+
+struct WindowsMemoryFunctions
+{
+  Common::DynamicLibrary m_kernel32_handle;
+  Common::DynamicLibrary m_api_ms_win_core_memory_l1_1_6_handle;
+  void* m_address_UnmapViewOfFileEx = nullptr;
+  void* m_address_VirtualAlloc2 = nullptr;
+  void* m_address_MapViewOfFile3 = nullptr;
+};
 #endif
 
 // This class lets you create a block of anonymous RAM, and then arbitrarily map views into it.
@@ -110,11 +119,7 @@ private:
   std::vector<WindowsMemoryRegion> m_regions;
   void* m_reserved_region = nullptr;
   void* m_memory_handle = nullptr;
-  Common::DynamicLibrary m_kernel32_handle;
-  Common::DynamicLibrary m_api_ms_win_core_memory_l1_1_6_handle;
-  void* m_address_UnmapViewOfFileEx = nullptr;
-  void* m_address_VirtualAlloc2 = nullptr;
-  void* m_address_MapViewOfFile3 = nullptr;
+  WindowsMemoryFunctions m_memory_functions;
 #else
   int m_shm_fd = 0;
   void* m_reserved_region = nullptr;
@@ -155,9 +160,34 @@ public:
   ///
   void Release();
 
+  ///
+  /// Ensure that the memory page at the given byte offset from the start of the memory region is
+  /// writable. We use this on Windows as a workaround to only actually commit pages as they are
+  /// written to. On other OSes this does nothing.
+  ///
+  /// @param offset The offset into the memory region that should be made writable if it isn't.
+  ///
+  void EnsureMemoryPageWritable(size_t offset)
+  {
+#ifdef _WIN32
+    const size_t block_index = offset / BLOCK_SIZE;
+    if (m_writable_block_handles[block_index] == nullptr)
+      MakeMemoryBlockWritable(block_index);
+#endif
+  }
+
 private:
   void* m_memory = nullptr;
   size_t m_size = 0;
+
+#ifdef _WIN32
+  void* m_zero_block = nullptr;
+  constexpr static size_t BLOCK_SIZE = 8 * 1024 * 1024;  // size of allocated memory blocks
+  WindowsMemoryFunctions m_memory_functions;
+  std::vector<void*> m_writable_block_handles;
+
+  void MakeMemoryBlockWritable(size_t offset);
+#endif
 };
 
 }  // namespace Common

Source/Core/Common/MemArenaWin.cpp

@@ -12,6 +12,7 @@
 
 #include <windows.h>
 
+#include "Common/Align.h"
 #include "Common/Assert.h"
 #include "Common/CommonFuncs.h"
 #include "Common/CommonTypes.h"
@@ -49,48 +50,55 @@ struct WindowsMemoryRegion
   }
 };
 
+static bool InitWindowsMemoryFunctions(WindowsMemoryFunctions* functions)
+{
+  DynamicLibrary kernelBase{"KernelBase.dll"};
+  if (!kernelBase.IsOpen())
+    return false;
+
+  void* const ptr_IsApiSetImplemented = kernelBase.GetSymbolAddress("IsApiSetImplemented");
+  if (!ptr_IsApiSetImplemented)
+    return false;
+  if (!static_cast<PIsApiSetImplemented>(ptr_IsApiSetImplemented)("api-ms-win-core-memory-l1-1-6"))
+    return false;
+
+  functions->m_api_ms_win_core_memory_l1_1_6_handle.Open("api-ms-win-core-memory-l1-1-6.dll");
+  functions->m_kernel32_handle.Open("Kernel32.dll");
+  if (!functions->m_api_ms_win_core_memory_l1_1_6_handle.IsOpen() ||
+      !functions->m_kernel32_handle.IsOpen())
+  {
+    functions->m_api_ms_win_core_memory_l1_1_6_handle.Close();
+    functions->m_kernel32_handle.Close();
+    return false;
+  }
+
+  void* const address_VirtualAlloc2 =
+      functions->m_api_ms_win_core_memory_l1_1_6_handle.GetSymbolAddress("VirtualAlloc2FromApp");
+  void* const address_MapViewOfFile3 =
+      functions->m_api_ms_win_core_memory_l1_1_6_handle.GetSymbolAddress("MapViewOfFile3FromApp");
+  void* const address_UnmapViewOfFileEx =
+      functions->m_kernel32_handle.GetSymbolAddress("UnmapViewOfFileEx");
+  if (address_VirtualAlloc2 && address_MapViewOfFile3 && address_UnmapViewOfFileEx)
+  {
+    functions->m_address_VirtualAlloc2 = address_VirtualAlloc2;
+    functions->m_address_MapViewOfFile3 = address_MapViewOfFile3;
+    functions->m_address_UnmapViewOfFileEx = address_UnmapViewOfFileEx;
+    return true;
+  }
+
+  // at least one function is not available, use legacy logic
+  functions->m_api_ms_win_core_memory_l1_1_6_handle.Close();
+  functions->m_kernel32_handle.Close();
+  return false;
+}
+
 MemArena::MemArena()
 {
   // Check if VirtualAlloc2 and MapViewOfFile3 are available, which provide functionality to reserve
   // a memory region no other allocation may occupy while still allowing us to allocate and map
   // stuff within it. If they're not available we'll instead fall back to the 'legacy' logic and
   // just hope that nothing allocates in our address range.
-  DynamicLibrary kernelBase{"KernelBase.dll"};
-  if (!kernelBase.IsOpen())
-    return;
-
-  void* const ptr_IsApiSetImplemented = kernelBase.GetSymbolAddress("IsApiSetImplemented");
-  if (!ptr_IsApiSetImplemented)
-    return;
-  if (!static_cast<PIsApiSetImplemented>(ptr_IsApiSetImplemented)("api-ms-win-core-memory-l1-1-6"))
-    return;
-
-  m_api_ms_win_core_memory_l1_1_6_handle.Open("api-ms-win-core-memory-l1-1-6.dll");
-  m_kernel32_handle.Open("Kernel32.dll");
-  if (!m_api_ms_win_core_memory_l1_1_6_handle.IsOpen() || !m_kernel32_handle.IsOpen())
-  {
-    m_api_ms_win_core_memory_l1_1_6_handle.Close();
-    m_kernel32_handle.Close();
-    return;
-  }
-
-  void* const address_VirtualAlloc2 =
-      m_api_ms_win_core_memory_l1_1_6_handle.GetSymbolAddress("VirtualAlloc2FromApp");
-  void* const address_MapViewOfFile3 =
-      m_api_ms_win_core_memory_l1_1_6_handle.GetSymbolAddress("MapViewOfFile3FromApp");
-  void* const address_UnmapViewOfFileEx = m_kernel32_handle.GetSymbolAddress("UnmapViewOfFileEx");
-  if (address_VirtualAlloc2 && address_MapViewOfFile3 && address_UnmapViewOfFileEx)
-  {
-    m_address_VirtualAlloc2 = address_VirtualAlloc2;
-    m_address_MapViewOfFile3 = address_MapViewOfFile3;
-    m_address_UnmapViewOfFileEx = address_UnmapViewOfFileEx;
-  }
-  else
-  {
-    // at least one function is not available, use legacy logic
-    m_api_ms_win_core_memory_l1_1_6_handle.Close();
-    m_kernel32_handle.Close();
-  }
+  InitWindowsMemoryFunctions(&m_memory_functions);
 }
 
 MemArena::~MemArena()
@@ -146,9 +154,9 @@ u8* MemArena::ReserveMemoryRegion(size_t memory_size)
   }
 
   u8* base;
-  if (m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
+  if (m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
   {
-    base = static_cast<u8*>(static_cast<PVirtualAlloc2>(m_address_VirtualAlloc2)(
+    base = static_cast<u8*>(static_cast<PVirtualAlloc2>(m_memory_functions.m_address_VirtualAlloc2)(
         nullptr, nullptr, memory_size, MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS,
         nullptr, 0));
     if (base)
@@ -177,7 +185,7 @@ u8* MemArena::ReserveMemoryRegion(size_t memory_size)
 
 void MemArena::ReleaseMemoryRegion()
 {
-  if (m_api_ms_win_core_memory_l1_1_6_handle.IsOpen() && m_reserved_region)
+  if (m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen() && m_reserved_region)
   {
     // user should have unmapped everything by this point, check if that's true and yell if not
     // (it indicates a bug in the emulated memory mapping logic)
@@ -314,7 +322,7 @@ WindowsMemoryRegion* MemArena::EnsureSplitRegionForMapping(void* start_address,
 
 void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base)
 {
-  if (m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
+  if (m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
   {
     WindowsMemoryRegion* const region = EnsureSplitRegionForMapping(base, size);
     if (!region)
@@ -323,7 +331,7 @@ void* MemArena::MapInMemoryRegion(s64 offset, size_t size, void* base)
       return nullptr;
     }
 
-    void* rv = static_cast<PMapViewOfFile3>(m_address_MapViewOfFile3)(
+    void* rv = static_cast<PMapViewOfFile3>(m_memory_functions.m_address_MapViewOfFile3)(
         m_memory_handle, nullptr, base, offset, size, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE,
         nullptr, 0);
     if (rv)
@@ -416,10 +424,10 @@ bool MemArena::JoinRegionsAfterUnmap(void* start_address, size_t size)
 
 void MemArena::UnmapFromMemoryRegion(void* view, size_t size)
 {
-  if (m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
+  if (m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
   {
-    if (static_cast<PUnmapViewOfFileEx>(m_address_UnmapViewOfFileEx)(view,
-                                                                     MEM_PRESERVE_PLACEHOLDER))
+    if (static_cast<PUnmapViewOfFileEx>(m_memory_functions.m_address_UnmapViewOfFileEx)(
+            view, MEM_PRESERVE_PLACEHOLDER))
     {
       if (!JoinRegionsAfterUnmap(view, size))
         PanicAlertFmt("Joining memory region failed.");
@@ -434,7 +442,10 @@ void MemArena::UnmapFromMemoryRegion(void* view, size_t size)
   UnmapViewOfFile(view);
 }
 
-LazyMemoryRegion::LazyMemoryRegion() = default;
+LazyMemoryRegion::LazyMemoryRegion()
+{
+  InitWindowsMemoryFunctions(&m_memory_functions);
+}
 
 LazyMemoryRegion::~LazyMemoryRegion()
 {
@@ -448,15 +459,67 @@ void* LazyMemoryRegion::Create(size_t size)
   if (size == 0)
     return nullptr;
 
-  void* memory = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (!m_memory_functions.m_api_ms_win_core_memory_l1_1_6_handle.IsOpen())
+    return nullptr;
+
+  // reserve block of memory
+  const size_t memory_size = Common::AlignUp(size, BLOCK_SIZE);
+  const size_t block_count = memory_size / BLOCK_SIZE;
+  u8* memory =
+      static_cast<u8*>(static_cast<PVirtualAlloc2>(m_memory_functions.m_address_VirtualAlloc2)(
+          nullptr, nullptr, memory_size, MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS,
+          nullptr, 0));
   if (!memory)
   {
-    NOTICE_LOG_FMT(MEMMAP, "Memory allocation of {} bytes failed.", size);
+    NOTICE_LOG_FMT(MEMMAP, "Memory reservation of {} bytes failed.", size);
     return nullptr;
   }
 
+  // split into individual block-sized regions
+  for (size_t i = 0; i < block_count - 1; ++i)
+  {
+    if (!VirtualFree(memory + i * BLOCK_SIZE, BLOCK_SIZE, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER))
+    {
+      NOTICE_LOG_FMT(MEMMAP, "Region splitting failed: {}", GetLastErrorString());
+
+      // release every split block as well as the remaining unsplit one
+      for (size_t j = 0; j < i + 1; ++j)
+        VirtualFree(memory + j * BLOCK_SIZE, 0, MEM_RELEASE);
+
+      return nullptr;
+    }
+  }
+
   m_memory = memory;
-  m_size = size;
+  m_size = memory_size;
+
+  // allocate a single block of real memory in the page file
+  HANDLE zero_block = CreateFileMapping(INVALID_HANDLE_VALUE, nullptr, PAGE_READONLY,
+                                        GetHighDWORD(BLOCK_SIZE), GetLowDWORD(BLOCK_SIZE), nullptr);
+  if (zero_block == nullptr)
+  {
+    NOTICE_LOG_FMT(MEMMAP, "CreateFileMapping() failed for zero block: {}", GetLastErrorString());
+    Release();
+    return nullptr;
+  }
+
+  m_zero_block = zero_block;
+
+  // map the zero page into every block
+  for (size_t i = 0; i < block_count; ++i)
+  {
+    void* result = static_cast<PMapViewOfFile3>(m_memory_functions.m_address_MapViewOfFile3)(
+        zero_block, nullptr, memory + i * BLOCK_SIZE, 0, BLOCK_SIZE, MEM_REPLACE_PLACEHOLDER,
+        PAGE_READONLY, nullptr, 0);
+    if (!result)
+    {
+      NOTICE_LOG_FMT(MEMMAP, "Mapping the zero block failed: {}", GetLastErrorString());
+      Release();
+      return nullptr;
+    }
+  }
+
+  m_writable_block_handles.resize(block_count, nullptr);
 
   return memory;
 }
@@ -464,19 +527,105 @@ void* LazyMemoryRegion::Create(size_t size)
 void LazyMemoryRegion::Clear()
 {
   ASSERT(m_memory);
+  u8* const memory = static_cast<u8*>(m_memory);
 
-  VirtualFree(m_memory, m_size, MEM_DECOMMIT);
-  VirtualAlloc(m_memory, m_size, MEM_COMMIT, PAGE_READWRITE);
+  // reset every writable block back to the zero block
+  for (size_t i = 0; i < m_writable_block_handles.size(); ++i)
+  {
+    if (m_writable_block_handles[i] == nullptr)
+      continue;
+
+    // unmap the writable block
+    if (!static_cast<PUnmapViewOfFileEx>(m_memory_functions.m_address_UnmapViewOfFileEx)(
+            memory + i * BLOCK_SIZE, MEM_PRESERVE_PLACEHOLDER))
+    {
+      PanicAlertFmt("Failed to unmap the writable block: {}", GetLastErrorString());
+    }
+
+    // free the writable block
+    if (!CloseHandle(m_writable_block_handles[i]))
+    {
+      PanicAlertFmt("Failed to free the writable block: {}", GetLastErrorString());
+    }
+    m_writable_block_handles[i] = nullptr;
+
+    // map the zero block
+    void* map_result = static_cast<PMapViewOfFile3>(m_memory_functions.m_address_MapViewOfFile3)(
+        m_zero_block, nullptr, memory + i * BLOCK_SIZE, 0, BLOCK_SIZE, MEM_REPLACE_PLACEHOLDER,
+        PAGE_READONLY, nullptr, 0);
+    if (!map_result)
+    {
+      PanicAlertFmt("Failed to re-map the zero block: {}", GetLastErrorString());
+    }
+  }
 }
 
 void LazyMemoryRegion::Release()
 {
   if (m_memory)
   {
-    VirtualFree(m_memory, 0, MEM_RELEASE);
+    // unmap all pages and release the not-zero block handles
+    u8* const memory = static_cast<u8*>(m_memory);
+    for (size_t i = 0; i < m_writable_block_handles.size(); ++i)
+    {
+      static_cast<PUnmapViewOfFileEx>(m_memory_functions.m_address_UnmapViewOfFileEx)(
+          memory + i * BLOCK_SIZE, MEM_PRESERVE_PLACEHOLDER);
+      if (m_writable_block_handles[i])
+      {
+        CloseHandle(m_writable_block_handles[i]);
+        m_writable_block_handles[i] = nullptr;
+      }
+    }
+  }
+  if (m_zero_block)
+  {
+    CloseHandle(m_zero_block);
+    m_zero_block = nullptr;
+  }
+  if (m_memory)
+  {
+    u8* const memory = static_cast<u8*>(m_memory);
+    const size_t block_count = m_size / BLOCK_SIZE;
+    for (size_t i = 0; i < block_count; ++i)
+      VirtualFree(memory + i * BLOCK_SIZE, 0, MEM_RELEASE);
     m_memory = nullptr;
     m_size = 0;
   }
 }
 
+void LazyMemoryRegion::MakeMemoryBlockWritable(size_t block_index)
+{
+  u8* const memory = static_cast<u8*>(m_memory);
+
+  // unmap the zero block
+  if (!static_cast<PUnmapViewOfFileEx>(m_memory_functions.m_address_UnmapViewOfFileEx)(
+          memory + block_index * BLOCK_SIZE, MEM_PRESERVE_PLACEHOLDER))
+  {
+    PanicAlertFmt("Failed to unmap the zero block: {}", GetLastErrorString());
+    return;
+  }
+
+  // allocate a fresh block to map
+  HANDLE block = CreateFileMapping(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE,
+                                   GetHighDWORD(BLOCK_SIZE), GetLowDWORD(BLOCK_SIZE), nullptr);
+  if (block == nullptr)
+  {
+    PanicAlertFmt("CreateFileMapping() failed for writable block: {}", GetLastErrorString());
+    return;
+  }
+
+  // map the new block
+  void* map_result = static_cast<PMapViewOfFile3>(m_memory_functions.m_address_MapViewOfFile3)(
+      block, nullptr, memory + block_index * BLOCK_SIZE, 0, BLOCK_SIZE, MEM_REPLACE_PLACEHOLDER,
+      PAGE_READWRITE, nullptr, 0);
+  if (!map_result)
+  {
+    PanicAlertFmt("Failed to map the writable block: {}", GetLastErrorString());
+    CloseHandle(block);
+    return;
+  }
+
+  m_writable_block_handles[block_index] = block;
+}
+
 }  // namespace Common

Source/Core/Core/PowerPC/JitCommon/JitCache.cpp

@@ -121,9 +121,14 @@ void JitBaseBlockCache::FinalizeBlock(JitBlock& block, bool block_link,
 {
   size_t index = FastLookupIndexForAddress(block.effectiveAddress, block.feature_flags);
   if (m_entry_points_ptr)
+  {
+    m_entry_points_arena.EnsureMemoryPageWritable(index * sizeof(u8*));
     m_entry_points_ptr[index] = block.normalEntry;
+  }
   else
+  {
     m_fast_block_map_fallback[index] = █
+  }
   block.fast_block_map_index = index;
 
   block.physical_addresses = physical_addresses;
@@ -485,9 +490,14 @@ JitBlock* JitBaseBlockCache::MoveBlockIntoFastCache(u32 addr, CPUEmuFeatureFlags
   // And create a new one
   size_t index = FastLookupIndexForAddress(addr, feature_flags);
   if (m_entry_points_ptr)
+  {
+    m_entry_points_arena.EnsureMemoryPageWritable(index * sizeof(u8*));
     m_entry_points_ptr[index] = block->normalEntry;
+  }
   else
+  {
     m_fast_block_map_fallback[index] = block;
+  }
   block->fast_block_map_index = index;
 
   return block;