diff --git a/Source/Core/Core/HW/SI_DeviceGCAdapter.cpp b/Source/Core/Core/HW/SI_DeviceGCAdapter.cpp
index e68176ac3e..2f955164bf 100644
--- a/Source/Core/Core/HW/SI_DeviceGCAdapter.cpp
+++ b/Source/Core/Core/HW/SI_DeviceGCAdapter.cpp
@@ -27,7 +27,12 @@ GCPadStatus CSIDevice_GCAdapter::GetPadStatus()
 	GCPadStatus PadStatus;
 	memset(&PadStatus, 0, sizeof(PadStatus));
 
-	GCAdapter::Input(ISIDevice::m_iDeviceNumber, &PadStatus);
+	// For netplay, the local controllers are polled in GetNetPads(), and
+	// the remote controllers receive their status there as well
+	if (!NetPlay::IsNetPlayRunning())
+	{
+		GCAdapter::Input(ISIDevice::m_iDeviceNumber, &PadStatus);
+	}
 
 	HandleMoviePadStatus(&PadStatus);
 
diff --git a/Source/Core/Core/HW/SI_DeviceGCController.cpp b/Source/Core/Core/HW/SI_DeviceGCController.cpp
index a6c25f689a..572e49c8b8 100644
--- a/Source/Core/Core/HW/SI_DeviceGCController.cpp
+++ b/Source/Core/Core/HW/SI_DeviceGCController.cpp
@@ -8,6 +8,7 @@
 #include "Common/Logging/Log.h"
 #include "Core/CoreTiming.h"
 #include "Core/Movie.h"
+#include "Core/NetPlayProto.h"
 #include "Core/HW/GCPad.h"
 #include "Core/HW/ProcessorInterface.h"
 #include "Core/HW/SI_Device.h"
@@ -144,7 +145,12 @@ GCPadStatus CSIDevice_GCController::GetPadStatus()
 	GCPadStatus PadStatus;
 	memset(&PadStatus, 0, sizeof(PadStatus));
 
-	Pad::GetStatus(ISIDevice::m_iDeviceNumber, &PadStatus);
+	// For netplay, the local controllers are polled in GetNetPads(), and
+	// the remote controllers receive their status there as well
+	if (!NetPlay::IsNetPlayRunning())
+	{
+		Pad::GetStatus(ISIDevice::m_iDeviceNumber, &PadStatus);
+	}
 
 	HandleMoviePadStatus(&PadStatus);
 	return PadStatus;
diff --git a/Source/Core/Core/NetPlayClient.cpp b/Source/Core/Core/NetPlayClient.cpp
index e292f16465..a8992e1a21 100644
--- a/Source/Core/Core/NetPlayClient.cpp
+++ b/Source/Core/Core/NetPlayClient.cpp
@@ -19,6 +19,7 @@
 #include "Core/HW/WiimoteEmu/WiimoteEmu.h"
 #include "Core/HW/WiimoteReal/WiimoteReal.h"
 #include "Core/IPC_HLE/WII_IPC_HLE_Device_usb.h"
+#include "InputCommon/GCAdapter.h"
 
 static std::mutex crit_netplay_client;
 static NetPlayClient * netplay_client = nullptr;
@@ -752,35 +753,34 @@ bool NetPlayClient::ChangeGame(const std::string&)
 void NetPlayClient::UpdateDevices()
 {
 	u8 local_pad = 0;
-	// Add local pads first:
-	// As stated in the comment in NetPlayClient::GetNetPads, the pads pertaining
-	// to the local user are always locally mapped to the first gamecube ports,
-	// so they should be added first.
+	u8 pad = 0;
+
 	for (auto player_id : m_pad_map)
 	{
 		// Use local controller types for local controllers if they are compatible
+		// Only GCController-like controllers are supported, GBA and similar
+		// exotic devices are not supported on netplay.
 		if (player_id == m_local_player->pid)
 		{
 			if (SIDevice_IsGCController(SConfig::GetInstance().m_SIDevice[local_pad]))
 			{
-				SerialInterface::AddDevice(SConfig::GetInstance().m_SIDevice[local_pad], local_pad);
+				SerialInterface::AddDevice(SConfig::GetInstance().m_SIDevice[local_pad], pad);
 			}
 			else
 			{
-				SerialInterface::AddDevice(SIDEVICE_GC_CONTROLLER, local_pad);
+				SerialInterface::AddDevice(SIDEVICE_GC_CONTROLLER, pad);
 			}
 			local_pad++;
 		}
-	}
-	for (auto player_id : m_pad_map)
-	{
-		if (player_id != m_local_player->pid)
+		else if (player_id > 0)
 		{
-			// Only GCController-like controllers are supported, GBA and similar
-			// exotic devices are not supported on netplay.
-			SerialInterface::AddDevice(player_id > 0 ? SIDEVICE_GC_CONTROLLER : SIDEVICE_NONE, local_pad);
-			local_pad++;
+			SerialInterface::AddDevice(SIDEVICE_GC_CONTROLLER, pad);
 		}
+		else
+		{
+			SerialInterface::AddDevice(SIDEVICE_NONE, pad);
+		}
+		pad++;
 	}
 }
 
@@ -863,39 +863,46 @@ bool NetPlayClient::GetNetPads(const u8 pad_nb, GCPadStatus* pad_status)
 	// If you have a 4P game, then one of the GameCubes will have
 	// a controller plugged into slot 1, and another in slot 2.
 	//
-	// The slot number is the "local" pad number, and what  player
+	// The slot number is the "local" pad number, and what player
 	// it actually means is the "in-game" pad number.
-	//
-	// The interface here gives us the status of local pads, and
-	// expects to get back "in-game" pad numbers back in response.
-	// e.g. it asks "here's the input that slot 1 has, and by the
-	// way, what's the state of P1?"
-	//
-	// We should add this split between "in-game" pads and "local"
-	// pads higher up.
 
-	int in_game_num = LocalPadToInGamePad(pad_nb);
-
-	// If this in-game pad is one of ours, then update from the
-	// information given.
-	if (in_game_num < 4)
+	// When the 1st in-game pad is polled, we assume the others will
+	// will be polled as well. To reduce latency, we poll all local
+	// controllers at once and then send the status to the other
+	// clients.
+	if (IsFirstInGamePad(pad_nb))
 	{
-		// adjust the buffer either up or down
-		// inserting multiple padstates or dropping states
-		while (m_pad_buffer[in_game_num].Size() <= m_target_buffer_size)
+		const u8 num_local_pads = NumLocalPads();
+		for (u8 local_pad = 0; local_pad < num_local_pads; local_pad++)
 		{
-			// add to buffer
-			m_pad_buffer[in_game_num].Push(*pad_status);
+			switch (SConfig::GetInstance().m_SIDevice[local_pad])
+			{
+			case SIDEVICE_WIIU_ADAPTER:
+				GCAdapter::Input(local_pad, pad_status);
+				break;
+			case SIDEVICE_GC_CONTROLLER:
+			default:
+				Pad::GetStatus(local_pad, pad_status);
+				break;
+			}
 
-			// send
-			SendPadState(in_game_num, *pad_status);
+			u8 ingame_pad = LocalPadToInGamePad(local_pad);
+
+			// adjust the buffer either up or down
+			// inserting multiple padstates or dropping states
+			while (m_pad_buffer[ingame_pad].Size() <= m_target_buffer_size)
+			{
+				// add to buffer
+				m_pad_buffer[ingame_pad].Push(*pad_status);
+
+				// send
+				SendPadState(ingame_pad, *pad_status);
+			}
 		}
 	}
 
-	// Now, we need to swap out the local value with the values
-	// retrieved from NetPlay. This could be the value we pushed
-	// above if we're configured as P1 and the code is trying
-	// to retrieve data for slot 1.
+	// Now, we either use the data pushed earlier, or wait for the
+	// other clients to send it to us
 	while (!m_pad_buffer[pad_nb].Pop(*pad_status))
 	{
 		if (!m_is_running.load())
@@ -1067,6 +1074,20 @@ bool NetPlayClient::LocalPlayerHasControllerMapped() const
 	       std::any_of(m_wiimote_map.begin(), m_wiimote_map.end(), mapping_matches_player_id);
 }
 
+bool NetPlayClient::IsFirstInGamePad(u8 ingame_pad) const
+{
+	return std::none_of(m_pad_map.begin(), m_pad_map.begin() + ingame_pad, [](auto mapping) {
+		return mapping > 0;
+	});
+}
+
+u8 NetPlayClient::NumLocalPads() const
+{
+	return static_cast<u8>(std::count_if(m_pad_map.begin(), m_pad_map.end(), [this](auto mapping) {
+		return mapping == m_local_player->pid;
+	}));
+}
+
 u8 NetPlayClient::InGamePadToLocalPad(u8 ingame_pad)
 {
 	// not our pad
diff --git a/Source/Core/Core/NetPlayClient.h b/Source/Core/Core/NetPlayClient.h
index 3c9222e087..285db13b5b 100644
--- a/Source/Core/Core/NetPlayClient.h
+++ b/Source/Core/Core/NetPlayClient.h
@@ -75,8 +75,11 @@ public:
 	void OnConnectReady(ENetAddress addr) override;
 	void OnConnectFailed(u8 reason) override;
 
+	bool IsFirstInGamePad(u8 ingame_pad) const;
+	u8 NumLocalPads() const;
+
+	u8 InGamePadToLocalPad(u8 ingame_pad);
 	u8 LocalPadToInGamePad(u8 localPad);
-	u8 InGamePadToLocalPad(u8 localPad);
 
 	u8 LocalWiimoteToInGameWiimote(u8 local_pad);