GDB stub (#1583)

* gdbstub beginnings

* gdbstub: finish gdb impl things, next up is integration with melonDS

* holy fuck the gdbstub works

* gdb breakpoints work, but there's a mysterious crash on continue

* fix memory corruption that sometimes happened, and make resetting the console thru gdb work

* remove some gdb debug printing

* fix things in gdbstub

* separate option for enabling gdbstub

* add mode-dependent CPU registers

* C++ize the GDBstub code

* add gdbstub config in emu settings dialog

* make sure gdb is disabled when jit is enabled

* Remove unnecessary compiler flags, mark ARMJIT assembly code as no-execute-stack

This hardens the binary a little bit against common exploitation methods

* add option to wait for debugger attach on startup

* only insert GNU stack notes on linux

* disable gdbstub enable checkbox when jit is enabled

* fix non-linux incompatibilities

* enable gdbstub by default

* fix issues with gdbstub settings disable stuff

* format stuff

* update gdb test code

* Fix segfault when calling StubCallbacks->GetCPU()

C++ overrides are hard. Please I'm just a lowly C programmer.

* fix packet size not being sent correctly

Thanks to @GlowingUmbreon on Github for troubleshooting this

* fix select(2) calls (i should read docs more properly)

* fix GDB command sequencing/parsing issue (hopefully)

* [GDB] implement no-ack mode

* fix sending ack on handshake

* get lldb to work

src/CMakeLists.txt

@@ -61,6 +61,15 @@ add_library(core STATIC
     tiny-AES-c/aes.c
     xxhash/xxhash.c)
 
+if (ENABLE_GDBSTUB)
+    message(NOTICE "Enabling GDB stub")
+    target_sources(core PRIVATE
+        debug/GdbStub.cpp
+        debug/GdbProto.cpp
+        debug/GdbCmds.cpp
+    )
+endif()
+
 if (ENABLE_OGLRENDERER)
     target_sources(core PRIVATE
         GPU_OpenGL.cpp
@@ -131,7 +140,7 @@ if (ENABLE_JIT)
 endif()
 
 if (WIN32)
-    target_link_libraries(core PRIVATE ole32 comctl32 ws2_32)
+    target_link_libraries(core PRIVATE ole32 comctl32 wsock32 ws2_32)
 elseif(NOT APPLE)
     check_library_exists(rt shm_open "" NEED_LIBRT)
     if (NEED_LIBRT)
@@ -143,3 +152,14 @@ if (ENABLE_JIT_PROFILING)
     target_include_directories(core PRIVATE "${VTUNE_INCLUDE_DIR}")
     target_link_libraries(core PRIVATE "${VTUNE_LIBRARY}")
 endif()
+
+#if(CMAKE_BUILD_TYPE MATCHES "Debug")
+#  set(
+#    CMAKE_C_FLAGS
+#    "${CMAKE_C_FLAGS} -fsanitize=undefined -fsanitize=address"
+#  )
+#  target_link_options(core
+#    BEFORE PUBLIC -fsanitize=undefined PUBLIC -fsanitize=address
+#  )
+#endif()
+