DSPHLE: Support EDuke32 Wii libaesnd uCode

This version is exclusive to EDuke32 Wii (see https://bugs.dolphin-emu.org/issues/12990).

Source/Core/Core/HW/DSPHLE/UCodes/AESnd.cpp

@@ -68,6 +68,16 @@ constexpr u32 ACCELERATOR_GAIN_8_BIT = 0x0100;
 // Multiply samples by 0x800/2048 = 1 (for ACCELERATOR_FORMAT_16_BIT)
 constexpr u32 ACCELERATOR_GAIN_16_BIT = 0x0800;
 
+bool AESndUCode::SwapLeftRight() const
+{
+  return m_crc == HASH_2012 || m_crc == HASH_EDUKE32 || m_crc == HASH_2020;
+}
+
+bool AESndUCode::UseNewFlagMasks() const
+{
+  return m_crc == HASH_EDUKE32 || m_crc == HASH_2020;
+}
+
 AESndUCode::AESndUCode(DSPHLE* dsphle, u32 crc) : UCodeInterface(dsphle, crc)
 {
 }
@@ -235,8 +245,8 @@ void AESndUCode::SetUpAccelerator(u16 format, [[maybe_unused]] u16 gain)
 
 void AESndUCode::DoMixing()
 {
-  const u32 pause_flag = (m_crc == HASH_2020) ? VOICE_PAUSE_NEW : VOICE_PAUSE_OLD;
+  const u32 pause_flag = UseNewFlagMasks() ? VOICE_PAUSE_NEW : VOICE_PAUSE_OLD;
-  const u32 format_mask = (m_crc == HASH_2020) ? VOICE_FORMAT_MASK_NEW : VOICE_FORMAT_MASK_OLD;
+  const u32 format_mask = UseNewFlagMasks() ? VOICE_FORMAT_MASK_NEW : VOICE_FORMAT_MASK_OLD;
   // dsp_mixer
   const bool paused = (m_parameter_block.flags & pause_flag) != 0;
   const bool running = (m_parameter_block.flags & VOICE_RUNNING) != 0;
@@ -246,6 +256,22 @@ void AESndUCode::DoMixing()
     // no_change_buffer
     const u32 voice_format = m_parameter_block.flags & format_mask;
     const bool is_16_bit = (voice_format & VOICE_16_BIT_FLAG) != 0;
+    if (m_crc == HASH_EDUKE32)
+    {
+      if (voice_format != VOICE_STEREO8 && voice_format != VOICE_STEREO16 &&
+          voice_format != VOICE_STEREO8_UNSIGNED)
+      {
+        // The EDuke32 Wii version does not support 16-but unsigned stereo, and also has broken
+        // handling of all mono formats.
+        if (!m_has_shown_unsupported_sample_format_warning)
+        {
+          m_has_shown_unsupported_sample_format_warning = true;
+          PanicAlertFmt("EDuke32 Wii aesndlib uCode does not correctly handle this sample format: "
+                        "{} (flags: {:08x})",
+                        voice_format, m_parameter_block.flags);
+        }
+      }
+    }
     // select_format table
     const u16 accelerator_format = is_16_bit ? ACCELERATOR_FORMAT_16_BIT : ACCELERATOR_FORMAT_8_BIT;
     const u16 accelerator_gain = is_16_bit ? ACCELERATOR_GAIN_16_BIT : ACCELERATOR_GAIN_8_BIT;
@@ -350,7 +376,7 @@ void AESndUCode::DoMixing()
           new_l ^= 0x8000;
           break;
         }
-        if (m_crc == HASH_2012 || m_crc == HASH_2020)
+        if (SwapLeftRight())
         {
           // The 2012 version swapped the left and right input channels so that left comes first,
           // and then right. Before, right came before left. The 2012 version didn't update comments

Source/Core/Core/HW/DSPHLE/UCodes/AESnd.h

@@ -31,6 +31,10 @@ public:
   // First included with libogc 1.8.11 on April 22, 2012: https://devkitpro.org/viewtopic.php?t=3094
   // https://github.com/devkitPro/libogc/commit/8f188e12b6a3d8b5a0d49a109fe6a3e4e1702aab
   static constexpr u32 HASH_2012 = 0x078066ab;
+  // Modified version used by EDuke32 Wii (padded to 0x03e0 bytes) - added unsigned 8-bit formats;
+  // broke the mono formats. The patch is based on the 2010 version, but it also includes the 2012
+  // input channel swap change. https://dukeworld.duke4.net/eduke32/wii/library_source_code/
+  static constexpr u32 HASH_EDUKE32 = 0x5ad4d933;
   // June 14, 2020 version (0x03e6 bytes) - added unsigned formats
   // First included with libogc 2.1.0 on June 15, 2020: https://devkitpro.org/viewtopic.php?t=9079
   // https://github.com/devkitPro/libogc/commit/eac8fe2c29aa790d552dd6166a1fb195dfdcb825
@@ -42,6 +46,9 @@ private:
   void SetUpAccelerator(u16 format, u16 gain);
   void DoMixing();
 
+  bool SwapLeftRight() const;
+  bool UseNewFlagMasks() const;
+
   // Copied from libaesnd/aesndlib.c's aesndpb_t (specifically the first 64 bytes)
 #pragma pack(1)
   struct ParameterBlock
@@ -83,5 +90,7 @@ private:
   static constexpr u32 NUM_OUTPUT_SAMPLES = 96;
 
   std::array<s16, NUM_OUTPUT_SAMPLES * 2> m_output_buffer{};
+
+  bool m_has_shown_unsupported_sample_format_warning = false;
 };
 }  // namespace DSP::HLE

Source/Core/Core/HW/DSPHLE/UCodes/UCodes.cpp

@@ -293,6 +293,7 @@ std::unique_ptr<UCodeInterface> UCodeFactory(u32 crc, DSPHLE* dsphle, bool wii)
 
   case AESndUCode::HASH_2010:
   case AESndUCode::HASH_2012:
+  case AESndUCode::HASH_EDUKE32:
   case AESndUCode::HASH_2020:
     INFO_LOG_FMT(DSPHLE, "CRC {:08x}: AESnd chosen (Homebrew)", crc);
     return std::make_unique<AESndUCode>(dsphle, crc);