diff --git a/CMakeLists.txt b/CMakeLists.txt
index f9d18f1cad..947fcd339b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -491,7 +491,6 @@ if(ENABLE_EVDEV)
     message(STATUS "libevdev/libudev found, enabling evdev controller backend")
     add_definitions(-DHAVE_LIBUDEV=1)
     add_definitions(-DHAVE_LIBEVDEV=1)
-    include_directories(${LIBUDEV_INCLUDE_DIR} ${LIBEVDEV_INCLUDE_DIR})
   else()
     message(FATAL_ERROR "Couldn't find libevdev and/or libudev. Can't build evdev controller backend.\nDisable ENABLE_EVDEV if you wish to build without controller support")
   endif()
diff --git a/Source/Core/InputCommon/CMakeLists.txt b/Source/Core/InputCommon/CMakeLists.txt
index 704ef394bf..6e44880a07 100644
--- a/Source/Core/InputCommon/CMakeLists.txt
+++ b/Source/Core/InputCommon/CMakeLists.txt
@@ -77,6 +77,11 @@ if(LIBEVDEV_FOUND AND LIBUDEV_FOUND)
   target_sources(inputcommon PRIVATE
     ControllerInterface/evdev/evdev.cpp
   )
+  target_include_directories(inputcommon
+  PRIVATE
+    ${LIBEVDEV_INCLUDE_DIR}
+    ${LIBUDEV_INCLUDE_DIR}
+  )
   target_link_libraries(inputcommon PUBLIC
     ${LIBEVDEV_LIBRARY}
     ${LIBUDEV_LIBRARY}