diff --git a/Source/Core/Common/Buffer.h b/Source/Core/Common/Buffer.h
index 216d0c5fe5..3915424fbf 100644
--- a/Source/Core/Common/Buffer.h
+++ b/Source/Core/Common/Buffer.h
@@ -40,6 +40,10 @@ public:
 
   void assign(PtrType ptr, size_type new_size) { BufferBase{std::move(ptr), new_size}.swap(*this); }
   void reset(size_type new_size = 0) { BufferBase{new_size}.swap(*this); }
+  void clear() { reset(); }
+
+  // Resize is purposely not provided as it often unnecessarily copies data about to be overwritten.
+  void resize(std::size_t) = delete;
 
   std::pair<PtrType, size_type> extract()
   {
diff --git a/Source/Core/Common/Image.cpp b/Source/Core/Common/Image.cpp
index bdb8dcb4a8..7fc1132e72 100644
--- a/Source/Core/Common/Image.cpp
+++ b/Source/Core/Common/Image.cpp
@@ -5,7 +5,6 @@
 
 #include <memory>
 #include <string>
-#include <vector>
 
 #include <spng.h>
 
@@ -28,7 +27,7 @@ static auto make_spng_ctx(int flags)
   return std::unique_ptr<spng_ctx, decltype(&spng_free)>(spng_ctx_new(flags), spng_free);
 }
 
-bool LoadPNG(const std::vector<u8>& input, std::vector<u8>* data_out, u32* width_out,
+bool LoadPNG(std::span<const u8> input, Common::UniqueBuffer<u8>* data_out, u32* width_out,
              u32* height_out)
 {
   auto ctx = make_spng_ctx(0);
@@ -47,7 +46,7 @@ bool LoadPNG(const std::vector<u8>& input, std::vector<u8>* data_out, u32* width
   if (spng_decoded_image_size(ctx.get(), format, &decoded_len))
     return false;
 
-  data_out->resize(decoded_len);
+  data_out->reset(decoded_len);
   if (spng_decode_image(ctx.get(), data_out->data(), decoded_len, format, SPNG_DECODE_TRNS))
     return false;
 
@@ -120,19 +119,20 @@ bool SavePNG(const std::string& path, const u8* input, ImageByteFormat format, u
   return true;
 }
 
-static std::vector<u8> RGBAToRGB(const u8* input, u32 width, u32 height, u32 row_stride)
+static Common::UniqueBuffer<u8> RGBAToRGB(const u8* input, u32 width, u32 height, u32 row_stride)
 {
-  std::vector<u8> buffer;
-  buffer.reserve(width * height * 3);
+  Common::UniqueBuffer<u8> buffer;
+  buffer.reset(width * height * 3);
 
+  std::size_t buffer_index = 0;
   for (u32 y = 0; y < height; ++y)
   {
     const u8* pos = input + y * row_stride;
     for (u32 x = 0; x < width; ++x)
     {
-      buffer.push_back(pos[x * 4]);
-      buffer.push_back(pos[x * 4 + 1]);
-      buffer.push_back(pos[x * 4 + 2]);
+      buffer[buffer_index++] = pos[x * 4];
+      buffer[buffer_index++] = pos[x * 4 + 1];
+      buffer[buffer_index++] = pos[x * 4 + 2];
     }
   }
   return buffer;
@@ -141,7 +141,7 @@ static std::vector<u8> RGBAToRGB(const u8* input, u32 width, u32 height, u32 row
 bool ConvertRGBAToRGBAndSavePNG(const std::string& path, const u8* input, u32 width, u32 height,
                                 u32 stride, int level)
 {
-  const std::vector<u8> data = RGBAToRGB(input, width, height, stride);
+  const auto data = RGBAToRGB(input, width, height, stride);
   return SavePNG(path, data.data(), ImageByteFormat::RGB, width, height, width * 3, level);
 }
 }  // namespace Common
diff --git a/Source/Core/Common/Image.h b/Source/Core/Common/Image.h
index 49303fd6af..3d2d846d17 100644
--- a/Source/Core/Common/Image.h
+++ b/Source/Core/Common/Image.h
@@ -3,14 +3,15 @@
 
 #pragma once
 
+#include <span>
 #include <string>
-#include <vector>
 
+#include "Common/Buffer.h"
 #include "Common/CommonTypes.h"
 
 namespace Common
 {
-bool LoadPNG(const std::vector<u8>& input, std::vector<u8>* data_out, u32* width_out,
+bool LoadPNG(std::span<const u8> input, Common::UniqueBuffer<u8>* data_out, u32* width_out,
              u32* height_out);
 
 enum class ImageByteFormat
diff --git a/Source/Core/DolphinQt/Achievements/AchievementBox.cpp b/Source/Core/DolphinQt/Achievements/AchievementBox.cpp
index 1fe85131cc..39eefb0484 100644
--- a/Source/Core/DolphinQt/Achievements/AchievementBox.cpp
+++ b/Source/Core/DolphinQt/Achievements/AchievementBox.cpp
@@ -85,7 +85,7 @@ void AchievementBox::UpdateData()
       color = AchievementManager::GOLD;
     else if (m_achievement->unlocked & RC_CLIENT_ACHIEVEMENT_UNLOCKED_SOFTCORE)
       color = AchievementManager::BLUE;
-    QImage i_badge(&badge.data.front(), badge.width, badge.height, QImage::Format_RGBA8888);
+    QImage i_badge(badge.data.data(), badge.width, badge.height, QImage::Format_RGBA8888);
     m_badge->setPixmap(
         QPixmap::fromImage(i_badge).scaled(64, 64, Qt::KeepAspectRatio, Qt::SmoothTransformation));
     m_badge->adjustSize();
diff --git a/Source/Core/InputCommon/ImageOperations.cpp b/Source/Core/InputCommon/ImageOperations.cpp
index 04293c9183..907cb52b8d 100644
--- a/Source/Core/InputCommon/ImageOperations.cpp
+++ b/Source/Core/InputCommon/ImageOperations.cpp
@@ -5,12 +5,9 @@
 
 #include <algorithm>
 #include <cmath>
-#include <limits>
-#include <stack>
 #include <string>
 #include <vector>
 
-#include "Common/FileUtil.h"
 #include "Common/IOFile.h"
 #include "Common/Image.h"
 
@@ -49,11 +46,11 @@ std::optional<ImagePixelData> LoadImage(const std::string& path)
 {
   File::IOFile file;
   file.Open(path, "rb");
-  std::vector<u8> buffer(file.GetSize());
+  Common::UniqueBuffer<u8> buffer(file.GetSize());
   file.ReadBytes(buffer.data(), file.GetSize());
 
   ImagePixelData image;
-  std::vector<u8> data;
+  Common::UniqueBuffer<u8> data;
   if (!Common::LoadPNG(buffer, &data, &image.width, &image.height))
     return std::nullopt;
 
@@ -74,19 +71,20 @@ std::optional<ImagePixelData> LoadImage(const std::string& path)
 
 bool WriteImage(const std::string& path, const ImagePixelData& image)
 {
-  std::vector<u8> buffer;
-  buffer.reserve(image.width * image.height * 4);
+  Common::UniqueBuffer<u8> buffer;
+  buffer.reset(image.width * image.height * 4);
 
+  std::size_t buffer_index = 0;
   for (u32 y = 0; y < image.height; ++y)
   {
     for (u32 x = 0; x < image.width; ++x)
     {
       const auto index = x + y * image.width;
       const auto pixel = image.pixels[index];
-      buffer.push_back(pixel.r);
-      buffer.push_back(pixel.g);
-      buffer.push_back(pixel.b);
-      buffer.push_back(pixel.a);
+      buffer[buffer_index++] = pixel.r;
+      buffer[buffer_index++] = pixel.g;
+      buffer[buffer_index++] = pixel.b;
+      buffer[buffer_index++] = pixel.a;
     }
   }
 
diff --git a/Source/Core/UICommon/GameFile.cpp b/Source/Core/UICommon/GameFile.cpp
index 66fb5a3a57..d548c55fbb 100644
--- a/Source/Core/UICommon/GameFile.cpp
+++ b/Source/Core/UICommon/GameFile.cpp
@@ -438,7 +438,7 @@ bool GameFile::ReadPNGBanner(const std::string& path)
     return false;
 
   GameBanner& banner = m_pending.custom_banner;
-  std::vector<u8> data_out;
+  Common::UniqueBuffer<u8> data_out;
   if (!Common::LoadPNG(png_data, &data_out, &banner.width, &banner.height))
     return false;
 
diff --git a/Source/Core/VideoCommon/Assets/CustomTextureData.cpp b/Source/Core/VideoCommon/Assets/CustomTextureData.cpp
index 8d21dc2139..1631002e1c 100644
--- a/Source/Core/VideoCommon/Assets/CustomTextureData.cpp
+++ b/Source/Core/VideoCommon/Assets/CustomTextureData.cpp
@@ -13,7 +13,6 @@
 #include "Common/IOFile.h"
 #include "Common/Image.h"
 #include "Common/Logging/Log.h"
-#include "Common/Swap.h"
 #include "VideoCommon/VideoConfig.h"
 
 namespace
@@ -235,7 +234,7 @@ static void ConvertTexture_X8R8G8B8(VideoCommon::CustomTextureData::ArraySlice::
 
 static void ConvertTexture_R8G8B8(VideoCommon::CustomTextureData::ArraySlice::Level* level)
 {
-  std::vector<u8> new_data(level->row_length * level->height * sizeof(u32));
+  Common::UniqueBuffer<u8> new_data(level->row_length * level->height * sizeof(u32));
 
   const u8* rgb_data_ptr = level->data.data();
   u8* data_ptr = new_data.data();
@@ -479,7 +478,7 @@ static bool ReadMipLevel(VideoCommon::CustomTextureData::ArraySlice::Level* leve
   level->height = height;
   level->format = info.format;
   level->row_length = row_length;
-  level->data.resize(size);
+  level->data.reset(size);
   if (!file.ReadBytes(level->data.data(), level->data.size()))
     return false;
 
@@ -571,13 +570,13 @@ bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, const std::stri
 
   File::IOFile file;
   file.Open(filename, "rb");
-  std::vector<u8> buffer(file.GetSize());
+  Common::UniqueBuffer<u8> buffer(file.GetSize());
   file.ReadBytes(buffer.data(), file.GetSize());
 
   return LoadPNGTexture(level, buffer);
 }
 
-bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, const std::vector<u8>& buffer)
+bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, std::span<const u8> buffer)
 {
   if (!level) [[unlikely]]
     return false;
diff --git a/Source/Core/VideoCommon/Assets/CustomTextureData.h b/Source/Core/VideoCommon/Assets/CustomTextureData.h
index 32607f4e88..048ee95923 100644
--- a/Source/Core/VideoCommon/Assets/CustomTextureData.h
+++ b/Source/Core/VideoCommon/Assets/CustomTextureData.h
@@ -3,9 +3,10 @@
 
 #pragma once
 
+#include <span>
 #include <string>
-#include <vector>
 
+#include "Common/Buffer.h"
 #include "Common/CommonTypes.h"
 #include "VideoCommon/TextureConfig.h"
 
@@ -18,7 +19,7 @@ public:
   {
     struct Level
     {
-      std::vector<u8> data;
+      Common::UniqueBuffer<u8> data;
       AbstractTextureFormat format = AbstractTextureFormat::RGBA8;
       u32 width = 0;
       u32 height = 0;
@@ -33,5 +34,5 @@ bool LoadDDSTexture(CustomTextureData* texture, const std::string& filename);
 bool LoadDDSTexture(CustomTextureData::ArraySlice::Level* level, const std::string& filename,
                     u32 mip_level);
 bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, const std::string& filename);
-bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, const std::vector<u8>& buffer);
+bool LoadPNGTexture(CustomTextureData::ArraySlice::Level* level, std::span<const u8> buffer);
 }  // namespace VideoCommon