Make netplay's "same game" check more robust

Instead of comparing the game ID, revision, disc number and name,
we can compare a hash of important parts of the disc including
all the aforementioned data but also additional data such as the
FST. The primary reason why I'm making this change is to let us
catch more desyncs before they happen, but this should also fix
https://bugs.dolphin-emu.org/issues/12115. As a bonus, the UI can
now distinguish the case where a client doesn't have the game at
all from the case where a client has the wrong version of the game.

Source/Core/DiscIO/DiscExtractor.cpp

@@ -245,19 +245,26 @@ bool ExportBI2Data(const Volume& volume, const Partition& partition,
   return ExportData(volume, partition, 0x440, 0x2000, export_filename);
 }
 
+std::optional<u64> GetApploaderSize(const Volume& volume, const Partition& partition)
+{
+  constexpr u64 header_size = 0x20;
+  const std::optional<u32> apploader_size = volume.ReadSwapped<u32>(0x2440 + 0x14, partition);
+  const std::optional<u32> trailer_size = volume.ReadSwapped<u32>(0x2440 + 0x18, partition);
+  if (!apploader_size || !trailer_size)
+    return std::nullopt;
+
+  return header_size + *apploader_size + *trailer_size;
+}
+
 bool ExportApploader(const Volume& volume, const Partition& partition,
                      const std::string& export_filename)
 {
   if (!IsDisc(volume.GetVolumeType()))
     return false;
 
-  std::optional<u32> apploader_size = volume.ReadSwapped<u32>(0x2440 + 0x14, partition);
-  const std::optional<u32> trailer_size = volume.ReadSwapped<u32>(0x2440 + 0x18, partition);
-  constexpr u32 header_size = 0x20;
-  if (!apploader_size || !trailer_size)
+  const std::optional<u64> apploader_size = GetApploaderSize(volume, partition);
+  if (!apploader_size)
     return false;
-  *apploader_size += *trailer_size + header_size;
-  DEBUG_LOG(DISCIO, "Apploader size -> %x", *apploader_size);
 
   return ExportData(volume, partition, 0x2440, *apploader_size, export_filename);
 }

Source/Core/DiscIO/DiscExtractor.h

@@ -61,6 +61,7 @@ bool ExportHeader(const Volume& volume, const Partition& partition,
                   const std::string& export_filename);
 bool ExportBI2Data(const Volume& volume, const Partition& partition,
                    const std::string& export_filename);
+std::optional<u64> GetApploaderSize(const Volume& volume, const Partition& partition);
 bool ExportApploader(const Volume& volume, const Partition& partition,
                      const std::string& export_filename);
 std::optional<u64> GetBootDOLOffset(const Volume& volume, const Partition& partition);

Source/Core/DiscIO/Volume.cpp

@@ -9,12 +9,16 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 
+#include "Core/IOS/ES/Formats.h"
 #include "DiscIO/Blob.h"
 #include "DiscIO/Enums.h"
 #include "DiscIO/VolumeDisc.h"
@@ -28,6 +32,43 @@ const IOS::ES::TicketReader Volume::INVALID_TICKET{};
 const IOS::ES::TMDReader Volume::INVALID_TMD{};
 const std::vector<u8> Volume::INVALID_CERT_CHAIN{};
 
+template <typename T>
+static void AddToSyncHash(mbedtls_sha1_context* context, const T& data)
+{
+  static_assert(std::is_trivially_copyable_v<T>);
+  mbedtls_sha1_update_ret(context, reinterpret_cast<const u8*>(&data), sizeof(data));
+}
+
+void Volume::ReadAndAddToSyncHash(mbedtls_sha1_context* context, u64 offset, u64 length,
+                                  const Partition& partition) const
+{
+  std::vector<u8> buffer(length);
+  if (Read(offset, length, buffer.data(), partition))
+    mbedtls_sha1_update_ret(context, buffer.data(), buffer.size());
+}
+
+void Volume::AddTMDToSyncHash(mbedtls_sha1_context* context, const Partition& partition) const
+{
+  // We want to hash some important parts of the TMD, but nothing that changes when fakesigning.
+  // (Fakesigned WADs are very popular, and we don't want people with properly signed WADs to
+  // unnecessarily be at a disadvantage due to most netplay partners having fakesigned WADs.)
+
+  const IOS::ES::TMDReader& tmd = GetTMD(partition);
+  if (!tmd.IsValid())
+    return;
+
+  AddToSyncHash(context, tmd.GetIOSId());
+  AddToSyncHash(context, tmd.GetTitleId());
+  AddToSyncHash(context, tmd.GetTitleFlags());
+  AddToSyncHash(context, tmd.GetGroupId());
+  AddToSyncHash(context, tmd.GetRegion());
+  AddToSyncHash(context, tmd.GetTitleVersion());
+  AddToSyncHash(context, tmd.GetBootIndex());
+
+  for (const IOS::ES::Content& content : tmd.GetContents())
+    AddToSyncHash(context, content);
+}
+
 std::map<Language, std::string> Volume::ReadWiiNames(const std::vector<char16_t>& data)
 {
   std::map<Language, std::string> names;

Source/Core/DiscIO/Volume.h

@@ -12,6 +12,8 @@
 #include <string>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 #include "Common/Swap.h"
@@ -138,6 +140,13 @@ public:
   virtual u64 GetRawSize() const = 0;
   virtual const BlobReader& GetBlobReader() const = 0;
 
+  // This hash is intended to be (but is not guaranteed to be):
+  // 1. Identical for discs with no differences that affect netplay/TAS sync
+  // 2. Different for discs with differences that affect netplay/TAS sync
+  // 3. Much faster than hashing the entire disc
+  // The way the hash is calculated may change with updates to Dolphin.
+  virtual std::array<u8, 20> GetSyncHash() const = 0;
+
 protected:
   template <u32 N>
   std::string DecodeString(const char (&data)[N]) const
@@ -151,6 +160,10 @@ protected:
       return CP1252ToUTF8(string);
   }
 
+  void ReadAndAddToSyncHash(mbedtls_sha1_context* context, u64 offset, u64 length,
+                            const Partition& partition) const;
+  void AddTMDToSyncHash(mbedtls_sha1_context* context, const Partition& partition) const;
+
   virtual u32 GetOffsetShift() const { return 0; }
   static std::map<Language, std::string> ReadWiiNames(const std::vector<char16_t>& data);
 

Source/Core/DiscIO/VolumeDisc.cpp

@@ -4,11 +4,17 @@
 
 #include "DiscIO/VolumeDisc.h"
 
+#include <memory>
 #include <optional>
 #include <string>
+#include <vector>
+
+#include <mbedtls/sha1.h>
 
 #include "Common/CommonTypes.h"
+#include "DiscIO/DiscExtractor.h"
 #include "DiscIO/Enums.h"
+#include "DiscIO/Filesystem.h"
 
 namespace DiscIO
 {
@@ -90,4 +96,35 @@ bool VolumeDisc::IsNKit() const
   return ReadSwapped<u32>(0x200, PARTITION_NONE) == NKIT_MAGIC;
 }
 
+void VolumeDisc::AddGamePartitionToSyncHash(mbedtls_sha1_context* context) const
+{
+  const Partition partition = GetGamePartition();
+
+  // All headers at the beginning of the partition, plus the apploader
+  ReadAndAddToSyncHash(context, 0, 0x2440 + GetApploaderSize(*this, partition).value_or(0),
+                       partition);
+
+  // Boot DOL (may be missing if this is a Datel disc)
+  const std::optional<u64> dol_offset = GetBootDOLOffset(*this, partition);
+  if (dol_offset)
+  {
+    ReadAndAddToSyncHash(context, *dol_offset,
+                         GetBootDOLSize(*this, partition, *dol_offset).value_or(0), partition);
+  }
+
+  // File system
+  const std::optional<u64> fst_offset = GetFSTOffset(*this, partition);
+  if (fst_offset)
+    ReadAndAddToSyncHash(context, *fst_offset, GetFSTSize(*this, partition).value_or(0), partition);
+
+  // opening.bnr (name and banner)
+  const FileSystem* file_system = GetFileSystem(partition);
+  if (file_system)
+  {
+    std::unique_ptr<FileInfo> file_info = file_system->FindFileInfo("opening.bnr");
+    if (file_info && !file_info->IsDirectory())
+      ReadAndAddToSyncHash(context, file_info->GetOffset(), file_info->GetSize(), partition);
+  }
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeDisc.h

@@ -7,6 +7,8 @@
 #include <optional>
 #include <string>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "DiscIO/Volume.h"
 
@@ -26,6 +28,7 @@ public:
 
 protected:
   Region RegionCodeToRegion(std::optional<u32> region_code) const;
+  void AddGamePartitionToSyncHash(mbedtls_sha1_context* context) const;
 };
 
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeGC.cpp

@@ -11,6 +11,8 @@
 #include <utility>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/Assert.h"
 #include "Common/ColorUtil.h"
 #include "Common/CommonTypes.h"
@@ -137,6 +139,19 @@ bool VolumeGC::IsDatelDisc() const
   return !GetBootDOLOffset(*this, PARTITION_NONE).has_value();
 }
 
+std::array<u8, 20> VolumeGC::GetSyncHash() const
+{
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  AddGamePartitionToSyncHash(&context);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 VolumeGC::ConvertedGCBanner VolumeGC::LoadBannerFile() const
 {
   GCBanner banner_file;

Source/Core/DiscIO/VolumeGC.h

@@ -51,6 +51,8 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
 
+  std::array<u8, 20> GetSyncHash() const override;
+
 private:
   static const u32 GC_BANNER_WIDTH = 96;
   static const u32 GC_BANNER_HEIGHT = 32;

Source/Core/DiscIO/VolumeWad.cpp

@@ -40,6 +40,7 @@ VolumeWAD::VolumeWAD(std::unique_ptr<BlobReader> reader) : m_reader(std::move(re
   m_ticket_size = m_reader->ReadSwapped<u32>(0x10).value_or(0);
   m_tmd_size = m_reader->ReadSwapped<u32>(0x14).value_or(0);
   m_data_size = m_reader->ReadSwapped<u32>(0x18).value_or(0);
+  m_opening_bnr_size = m_reader->ReadSwapped<u32>(0x1C).value_or(0);
 
   m_cert_chain_offset = Common::AlignUp(m_hdr_size, 0x40);
   m_ticket_offset = m_cert_chain_offset + Common::AlignUp(m_cert_chain_size, 0x40);
@@ -342,4 +343,22 @@ const BlobReader& VolumeWAD::GetBlobReader() const
   return *m_reader;
 }
 
+std::array<u8, 20> VolumeWAD::GetSyncHash() const
+{
+  // We can skip hashing the contents since the TMD contains hashes of the contents.
+  // We specifically don't hash the ticket, since its console ID can differ without any problems.
+
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  AddTMDToSyncHash(&context, PARTITION_NONE);
+
+  ReadAndAddToSyncHash(&context, m_opening_bnr_offset, m_opening_bnr_size, PARTITION_NONE);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWad.h

@@ -70,6 +70,8 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
 
+  std::array<u8, 20> GetSyncHash() const override;
+
 private:
   std::unique_ptr<BlobReader> m_reader;
   IOS::ES::TicketReader m_ticket;
@@ -85,6 +87,7 @@ private:
   u32 m_ticket_size = 0;
   u32 m_tmd_size = 0;
   u32 m_data_size = 0;
+  u32 m_opening_bnr_size = 0;
 };
 
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.cpp

@@ -364,6 +364,33 @@ const BlobReader& VolumeWii::GetBlobReader() const
   return *m_reader;
 }
 
+std::array<u8, 20> VolumeWii::GetSyncHash() const
+{
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  // Disc header
+  ReadAndAddToSyncHash(&context, 0, 0x80, PARTITION_NONE);
+
+  // Region code
+  ReadAndAddToSyncHash(&context, 0x4E000, 4, PARTITION_NONE);
+
+  // The data offset of the game partition - an important factor for disc drive timings
+  const u64 data_offset = PartitionOffsetToRawOffset(0, GetGamePartition());
+  mbedtls_sha1_update_ret(&context, reinterpret_cast<const u8*>(&data_offset), sizeof(data_offset));
+
+  // TMD
+  AddTMDToSyncHash(&context, GetGamePartition());
+
+  // Game partition contents
+  AddGamePartitionToSyncHash(&context);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const
 {
   auto it = m_partitions.find(partition);

Source/Core/DiscIO/VolumeWii.h

@@ -92,6 +92,7 @@ public:
   bool IsSizeAccurate() const override;
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
+  std::array<u8, 20> GetSyncHash() const override;
 
   // The in parameter can either contain all the data to begin with,
   // or read_function can write data into the in parameter when called.