diff --git a/Source/Core/InputCommon/GCAdapter.cpp b/Source/Core/InputCommon/GCAdapter.cpp
index 209594a59c..f567335223 100644
--- a/Source/Core/InputCommon/GCAdapter.cpp
+++ b/Source/Core/InputCommon/GCAdapter.cpp
@@ -60,7 +60,8 @@ static std::mutex s_mutex;
 static u8 s_controller_payload[37];
 static u8 s_controller_payload_swap[37];
 
-static std::atomic<int> s_controller_payload_size = {0};
+// Only access with s_mutex held!
+static int s_controller_payload_size = {0};
 
 static std::thread s_adapter_input_thread;
 static std::thread s_adapter_output_thread;
@@ -112,7 +113,7 @@ static void Read()
     {
       std::lock_guard<std::mutex> lk(s_mutex);
       std::swap(s_controller_payload_swap, s_controller_payload);
-      s_controller_payload_size.store(payload_size);
+      s_controller_payload_size = payload_size;
     }
 
     Common::YieldCPU();
@@ -478,7 +479,7 @@ GCPadStatus Input(int chan)
     std::lock_guard<std::mutex> lk(s_mutex);
     std::copy(std::begin(s_controller_payload), std::end(s_controller_payload),
               std::begin(controller_payload_copy));
-    payload_size = s_controller_payload_size.load();
+    payload_size = s_controller_payload_size;
   }
 
   GCPadStatus pad = {};
diff --git a/Source/Core/InputCommon/GCAdapter_Android.cpp b/Source/Core/InputCommon/GCAdapter_Android.cpp
index 1e58733236..499a9ece30 100644
--- a/Source/Core/InputCommon/GCAdapter_Android.cpp
+++ b/Source/Core/InputCommon/GCAdapter_Android.cpp
@@ -41,7 +41,7 @@ static u8 s_controller_rumble[4];
 // Input handling
 static std::mutex s_read_mutex;
 static std::array<u8, 37> s_controller_payload;
-static std::atomic<int> s_controller_payload_size{0};
+static int s_controller_payload_size{0};
 
 // Output handling
 static std::mutex s_write_mutex;
@@ -164,7 +164,7 @@ static void Read()
       {
         std::lock_guard<std::mutex> lk(s_read_mutex);
         std::copy(java_data, java_data + s_controller_payload.size(), s_controller_payload.begin());
-        s_controller_payload_size.store(read_size);
+        s_controller_payload_size = read_size;
       }
       env->ReleaseByteArrayElements(*java_controller_payload, java_data, 0);
 
@@ -284,7 +284,7 @@ GCPadStatus Input(int chan)
   {
     std::lock_guard<std::mutex> lk(s_read_mutex);
     controller_payload_copy = s_controller_payload;
-    payload_size = s_controller_payload_size.load();
+    payload_size = s_controller_payload_size;
   }
 
   GCPadStatus pad = {};
@@ -406,7 +406,7 @@ void ResetRumble()
 {
   unsigned char rumble[5] = {0x11, 0, 0, 0, 0};
   {
-    std::lock_guard<std::mutex> lk(s_read_mutex);
+    std::lock_guard<std::mutex> lk(s_write_mutex);
     memcpy(s_controller_write_payload, rumble, 5);
     s_controller_write_payload_size.store(5);
   }