Core: Abide by COM MTA requirement for cubeb on Windows. Partially based on https://github.com/dolphin-emu/dolphin/pull/8920#discussion_r459746604

Co-authored-by: Michael M <mchtly@gmail.com>

Source/Core/AudioCommon/CubebStream.cpp

@@ -66,20 +66,26 @@ bool CubebStream::Init()
 
 bool CubebStream::SetRunning(bool running)
 {
+  bool return_value = false;
+  CubebUtils::RunInCubebContext([&] {
     if (running)
-    return cubeb_stream_start(m_stream) == CUBEB_OK;
+      return_value = cubeb_stream_start(m_stream) == CUBEB_OK;
     else
-    return cubeb_stream_stop(m_stream) == CUBEB_OK;
+      return_value = cubeb_stream_stop(m_stream) == CUBEB_OK;
+  });
+  return return_value;
 }
 
 CubebStream::~CubebStream()
 {
+  CubebUtils::RunInCubebContext([&] {
     SetRunning(false);
     cubeb_stream_destroy(m_stream);
+  });
   m_ctx.reset();
 }
 
 void CubebStream::SetVolume(int volume)
 {
-  cubeb_stream_set_volume(m_stream, volume / 100.0f);
+  CubebUtils::RunInCubebContext([&] { cubeb_stream_set_volume(m_stream, volume / 100.0f); });
 }

Source/Core/AudioCommon/CubebUtils.cpp

@@ -6,7 +6,9 @@
 #include <cstdarg>
 #include <cstddef>
 #include <cstring>
+#include <thread>
 
+#include "Common/Assert.h"
 #include "Common/CommonPaths.h"
 #include "Common/Logging/Log.h"
 #include "Common/Logging/LogManager.h"
@@ -14,6 +16,49 @@
 
 #include <cubeb/cubeb.h>
 
+#ifdef _WIN32
+#include <Objbase.h>
+#endif
+
+// On Windows, we must manually ensure that COM is initialized in MTA mode on every thread that
+// accesses the cubeb API. See the comment on cubeb_init in cubeb.h
+// We do this with a thread-local variable that keeps track of whether COM is initialized or not,
+// and initialize it if it isn't. When the thread ends COM is uninitialized again.
+#ifdef _WIN32
+namespace
+{
+class auto_com
+{
+public:
+  auto_com() = default;
+  auto_com(const auto_com&) = delete;
+  auto_com(auto_com&&) = delete;
+  auto_com& operator=(const auto_com&) = delete;
+  auto_com& operator=(auto_com&&) = delete;
+  ~auto_com()
+  {
+    if (m_initialized)
+    {
+      CoUninitialize();
+    }
+  }
+  bool initialize()
+  {
+    if (!m_initialized)
+    {
+      HRESULT result = CoInitializeEx(nullptr, COINIT_MULTITHREADED | COINIT_DISABLE_OLE1DDE);
+      m_initialized = SUCCEEDED(result);
+    }
+    return m_initialized;
+  }
+
+private:
+  bool m_initialized = false;
+};
+}  // namespace
+static thread_local auto_com tls_com_context;
+#endif
+
 static ptrdiff_t s_path_cutoff_point = 0;
 
 static void LogCallback(const char* format, ...)
@@ -47,8 +92,38 @@ static void DestroyContext(cubeb* ctx)
   }
 }
 
+static bool EnsureCubebCallable()
+{
+#ifdef _WIN32
+  if (!tls_com_context.initialize())
+    return false;
+#endif
+  return true;
+}
+
+void CubebUtils::RunInCubebContext(const std::function<void()>& func)
+{
+  // Cubeb is documented to require MTA COM mode, so if the current thread was initialized in STA
+  // mode, we make a temporary thread to execute the cubeb call.
+  if (EnsureCubebCallable())
+  {
+    func();
+  }
+  else
+  {
+    std::thread([&] {
+      // this should never fail, so yell loudly if it does
+      ASSERT(EnsureCubebCallable());
+      func();
+    }).join();
+  }
+}
+
 std::shared_ptr<cubeb> CubebUtils::GetContext()
 {
+  if (!EnsureCubebCallable())
+    return nullptr;
+
   static std::weak_ptr<cubeb> weak;
 
   std::shared_ptr<cubeb> shared = weak.lock();

Source/Core/AudioCommon/CubebUtils.h

@@ -3,11 +3,13 @@
 
 #pragma once
 
+#include <functional>
 #include <memory>
 
 struct cubeb;
 
 namespace CubebUtils
 {
+void RunInCubebContext(const std::function<void()>& func);
 std::shared_ptr<cubeb> GetContext();
 }  // namespace CubebUtils

Source/Core/Core/HW/EXI/EXI_DeviceMic.cpp

@@ -68,11 +68,12 @@ void CEXIMic::StreamStart()
   if (!m_cubeb_ctx)
     return;
 
+  CubebUtils::RunInCubebContext([&] {
     // Open stream with current parameters
     stream_size = buff_size_samples * 500;
     stream_buffer = new s16[stream_size];
 
-  cubeb_stream_params params;
+    cubeb_stream_params params{};
     params.format = CUBEB_SAMPLE_S16LE;
     params.rate = sample_rate;
     params.channels = 1;
@@ -84,9 +85,9 @@ void CEXIMic::StreamStart()
       WARN_LOG_FMT(EXPANSIONINTERFACE, "Error getting minimum latency");
     }
 
-  if (cubeb_stream_init(m_cubeb_ctx.get(), &m_cubeb_stream, "Dolphin Emulated GameCube Microphone",
-                        nullptr, &params, nullptr, nullptr,
-                        std::max<u32>(buff_size_samples, minimum_latency), DataCallback,
+    if (cubeb_stream_init(m_cubeb_ctx.get(), &m_cubeb_stream,
+                          "Dolphin Emulated GameCube Microphone", nullptr, &params, nullptr,
+                          nullptr, std::max<u32>(buff_size_samples, minimum_latency), DataCallback,
                           state_callback, this) != CUBEB_OK)
     {
       ERROR_LOG_FMT(EXPANSIONINTERFACE, "Error initializing cubeb stream");
@@ -100,15 +101,18 @@ void CEXIMic::StreamStart()
     }
 
     INFO_LOG_FMT(EXPANSIONINTERFACE, "started cubeb stream");
+  });
 }
 
 void CEXIMic::StreamStop()
 {
   if (m_cubeb_stream)
   {
+    CubebUtils::RunInCubebContext([&] {
       if (cubeb_stream_stop(m_cubeb_stream) != CUBEB_OK)
         ERROR_LOG_FMT(EXPANSIONINTERFACE, "Error stopping cubeb stream");
       cubeb_stream_destroy(m_cubeb_stream);
+    });
     m_cubeb_stream = nullptr;
   }