Merge pull request #8861 from JosJuice/netplay-hash

Make netplay's "same game" check more robust

Source/Core/DiscIO/DiscExtractor.cpp

@@ -245,19 +245,26 @@ bool ExportBI2Data(const Volume& volume, const Partition& partition,
   return ExportData(volume, partition, 0x440, 0x2000, export_filename);
 }
 
+std::optional<u64> GetApploaderSize(const Volume& volume, const Partition& partition)
+{
+  constexpr u64 header_size = 0x20;
+  const std::optional<u32> apploader_size = volume.ReadSwapped<u32>(0x2440 + 0x14, partition);
+  const std::optional<u32> trailer_size = volume.ReadSwapped<u32>(0x2440 + 0x18, partition);
+  if (!apploader_size || !trailer_size)
+    return std::nullopt;
+
+  return header_size + *apploader_size + *trailer_size;
+}
+
 bool ExportApploader(const Volume& volume, const Partition& partition,
                      const std::string& export_filename)
 {
   if (!IsDisc(volume.GetVolumeType()))
     return false;
 
-  std::optional<u32> apploader_size = volume.ReadSwapped<u32>(0x2440 + 0x14, partition);
-  const std::optional<u32> trailer_size = volume.ReadSwapped<u32>(0x2440 + 0x18, partition);
-  constexpr u32 header_size = 0x20;
-  if (!apploader_size || !trailer_size)
+  const std::optional<u64> apploader_size = GetApploaderSize(volume, partition);
+  if (!apploader_size)
     return false;
-  *apploader_size += *trailer_size + header_size;
-  DEBUG_LOG(DISCIO, "Apploader size -> %x", *apploader_size);
 
   return ExportData(volume, partition, 0x2440, *apploader_size, export_filename);
 }

Source/Core/DiscIO/DiscExtractor.h

@@ -61,6 +61,7 @@ bool ExportHeader(const Volume& volume, const Partition& partition,
                   const std::string& export_filename);
 bool ExportBI2Data(const Volume& volume, const Partition& partition,
                    const std::string& export_filename);
+std::optional<u64> GetApploaderSize(const Volume& volume, const Partition& partition);
 bool ExportApploader(const Volume& volume, const Partition& partition,
                      const std::string& export_filename);
 std::optional<u64> GetBootDOLOffset(const Volume& volume, const Partition& partition);

Source/Core/DiscIO/Volume.cpp

@@ -9,12 +9,16 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 
+#include "Core/IOS/ES/Formats.h"
 #include "DiscIO/Blob.h"
 #include "DiscIO/Enums.h"
 #include "DiscIO/VolumeDisc.h"
@@ -28,6 +32,43 @@ const IOS::ES::TicketReader Volume::INVALID_TICKET{};
 const IOS::ES::TMDReader Volume::INVALID_TMD{};
 const std::vector<u8> Volume::INVALID_CERT_CHAIN{};
 
+template <typename T>
+static void AddToSyncHash(mbedtls_sha1_context* context, const T& data)
+{
+  static_assert(std::is_trivially_copyable_v<T>);
+  mbedtls_sha1_update_ret(context, reinterpret_cast<const u8*>(&data), sizeof(data));
+}
+
+void Volume::ReadAndAddToSyncHash(mbedtls_sha1_context* context, u64 offset, u64 length,
+                                  const Partition& partition) const
+{
+  std::vector<u8> buffer(length);
+  if (Read(offset, length, buffer.data(), partition))
+    mbedtls_sha1_update_ret(context, buffer.data(), buffer.size());
+}
+
+void Volume::AddTMDToSyncHash(mbedtls_sha1_context* context, const Partition& partition) const
+{
+  // We want to hash some important parts of the TMD, but nothing that changes when fakesigning.
+  // (Fakesigned WADs are very popular, and we don't want people with properly signed WADs to
+  // unnecessarily be at a disadvantage due to most netplay partners having fakesigned WADs.)
+
+  const IOS::ES::TMDReader& tmd = GetTMD(partition);
+  if (!tmd.IsValid())
+    return;
+
+  AddToSyncHash(context, tmd.GetIOSId());
+  AddToSyncHash(context, tmd.GetTitleId());
+  AddToSyncHash(context, tmd.GetTitleFlags());
+  AddToSyncHash(context, tmd.GetGroupId());
+  AddToSyncHash(context, tmd.GetRegion());
+  AddToSyncHash(context, tmd.GetTitleVersion());
+  AddToSyncHash(context, tmd.GetBootIndex());
+
+  for (const IOS::ES::Content& content : tmd.GetContents())
+    AddToSyncHash(context, content);
+}
+
 std::map<Language, std::string> Volume::ReadWiiNames(const std::vector<char16_t>& data)
 {
   std::map<Language, std::string> names;

Source/Core/DiscIO/Volume.h

@@ -12,6 +12,8 @@
 #include <string>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 #include "Common/Swap.h"
@@ -143,6 +145,13 @@ public:
   virtual u64 GetRawSize() const = 0;
   virtual const BlobReader& GetBlobReader() const = 0;
 
+  // This hash is intended to be (but is not guaranteed to be):
+  // 1. Identical for discs with no differences that affect netplay/TAS sync
+  // 2. Different for discs with differences that affect netplay/TAS sync
+  // 3. Much faster than hashing the entire disc
+  // The way the hash is calculated may change with updates to Dolphin.
+  virtual std::array<u8, 20> GetSyncHash() const = 0;
+
 protected:
   template <u32 N>
   std::string DecodeString(const char (&data)[N]) const
@@ -156,6 +165,10 @@ protected:
       return CP1252ToUTF8(string);
   }
 
+  void ReadAndAddToSyncHash(mbedtls_sha1_context* context, u64 offset, u64 length,
+                            const Partition& partition) const;
+  void AddTMDToSyncHash(mbedtls_sha1_context* context, const Partition& partition) const;
+
   virtual u32 GetOffsetShift() const { return 0; }
   static std::map<Language, std::string> ReadWiiNames(const std::vector<char16_t>& data);
 

Source/Core/DiscIO/VolumeDisc.cpp

@@ -4,11 +4,17 @@
 
 #include "DiscIO/VolumeDisc.h"
 
+#include <memory>
 #include <optional>
 #include <string>
+#include <vector>
+
+#include <mbedtls/sha1.h>
 
 #include "Common/CommonTypes.h"
+#include "DiscIO/DiscExtractor.h"
 #include "DiscIO/Enums.h"
+#include "DiscIO/Filesystem.h"
 
 namespace DiscIO
 {
@@ -90,4 +96,35 @@ bool VolumeDisc::IsNKit() const
   return ReadSwapped<u32>(0x200, PARTITION_NONE) == NKIT_MAGIC;
 }
 
+void VolumeDisc::AddGamePartitionToSyncHash(mbedtls_sha1_context* context) const
+{
+  const Partition partition = GetGamePartition();
+
+  // All headers at the beginning of the partition, plus the apploader
+  ReadAndAddToSyncHash(context, 0, 0x2440 + GetApploaderSize(*this, partition).value_or(0),
+                       partition);
+
+  // Boot DOL (may be missing if this is a Datel disc)
+  const std::optional<u64> dol_offset = GetBootDOLOffset(*this, partition);
+  if (dol_offset)
+  {
+    ReadAndAddToSyncHash(context, *dol_offset,
+                         GetBootDOLSize(*this, partition, *dol_offset).value_or(0), partition);
+  }
+
+  // File system
+  const std::optional<u64> fst_offset = GetFSTOffset(*this, partition);
+  if (fst_offset)
+    ReadAndAddToSyncHash(context, *fst_offset, GetFSTSize(*this, partition).value_or(0), partition);
+
+  // opening.bnr (name and banner)
+  const FileSystem* file_system = GetFileSystem(partition);
+  if (file_system)
+  {
+    std::unique_ptr<FileInfo> file_info = file_system->FindFileInfo("opening.bnr");
+    if (file_info && !file_info->IsDirectory())
+      ReadAndAddToSyncHash(context, file_info->GetOffset(), file_info->GetSize(), partition);
+  }
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeDisc.h

@@ -7,6 +7,8 @@
 #include <optional>
 #include <string>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/CommonTypes.h"
 #include "DiscIO/Volume.h"
 
@@ -26,6 +28,7 @@ public:
 
 protected:
   Region RegionCodeToRegion(std::optional<u32> region_code) const;
+  void AddGamePartitionToSyncHash(mbedtls_sha1_context* context) const;
 };
 
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeGC.cpp

@@ -11,6 +11,8 @@
 #include <utility>
 #include <vector>
 
+#include <mbedtls/sha1.h>
+
 #include "Common/Assert.h"
 #include "Common/ColorUtil.h"
 #include "Common/CommonTypes.h"
@@ -137,6 +139,19 @@ bool VolumeGC::IsDatelDisc() const
   return !GetBootDOLOffset(*this, PARTITION_NONE).has_value();
 }
 
+std::array<u8, 20> VolumeGC::GetSyncHash() const
+{
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  AddGamePartitionToSyncHash(&context);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 VolumeGC::ConvertedGCBanner VolumeGC::LoadBannerFile() const
 {
   GCBanner banner_file;

Source/Core/DiscIO/VolumeGC.h

@@ -51,6 +51,8 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
 
+  std::array<u8, 20> GetSyncHash() const override;
+
 private:
   static const u32 GC_BANNER_WIDTH = 96;
   static const u32 GC_BANNER_HEIGHT = 32;

Source/Core/DiscIO/VolumeVerifier.cpp

@@ -479,8 +479,8 @@ std::vector<Partition> VolumeVerifier::CheckPartitions()
       AddProblem(Severity::Low,
                  Common::GetStringT(
                      "The data partition is not at its normal position. This will affect the "
-                     "emulated loading times. When using NetPlay or sending input recordings to "
-                     "other people, you will experience desyncs if anyone is using a good dump."));
+                     "emulated loading times. You will be unable to share input recordings and use "
+                     "NetPlay with anyone who is using a good dump."));
     }
   }
 
@@ -783,10 +783,10 @@ void VolumeVerifier::CheckDiscSize(const std::vector<Partition>& partitions)
         {
           AddProblem(
               Severity::Low,
-              Common::GetStringT("This disc image has an unusual size. This will likely make the "
-                                 "emulated loading times longer. When using NetPlay or sending "
-                                 "input recordings to other people, you will likely experience "
-                                 "desyncs if anyone is using a good dump."));
+              Common::GetStringT(
+                  "This disc image has an unusual size. This will likely make the emulated "
+                  "loading times longer. You will likely be unable to share input recordings "
+                  "and use NetPlay with anyone who is using a good dump."));
         }
         else
         {

Source/Core/DiscIO/VolumeWad.cpp

@@ -40,6 +40,7 @@ VolumeWAD::VolumeWAD(std::unique_ptr<BlobReader> reader) : m_reader(std::move(re
   m_ticket_size = m_reader->ReadSwapped<u32>(0x10).value_or(0);
   m_tmd_size = m_reader->ReadSwapped<u32>(0x14).value_or(0);
   m_data_size = m_reader->ReadSwapped<u32>(0x18).value_or(0);
+  m_opening_bnr_size = m_reader->ReadSwapped<u32>(0x1C).value_or(0);
 
   m_cert_chain_offset = Common::AlignUp(m_hdr_size, 0x40);
   m_ticket_offset = m_cert_chain_offset + Common::AlignUp(m_cert_chain_size, 0x40);
@@ -342,4 +343,22 @@ const BlobReader& VolumeWAD::GetBlobReader() const
   return *m_reader;
 }
 
+std::array<u8, 20> VolumeWAD::GetSyncHash() const
+{
+  // We can skip hashing the contents since the TMD contains hashes of the contents.
+  // We specifically don't hash the ticket, since its console ID can differ without any problems.
+
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  AddTMDToSyncHash(&context, PARTITION_NONE);
+
+  ReadAndAddToSyncHash(&context, m_opening_bnr_offset, m_opening_bnr_size, PARTITION_NONE);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWad.h

@@ -70,6 +70,8 @@ public:
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
 
+  std::array<u8, 20> GetSyncHash() const override;
+
 private:
   std::unique_ptr<BlobReader> m_reader;
   IOS::ES::TicketReader m_ticket;
@@ -85,6 +87,7 @@ private:
   u32 m_ticket_size = 0;
   u32 m_tmd_size = 0;
   u32 m_data_size = 0;
+  u32 m_opening_bnr_size = 0;
 };
 
 }  // namespace DiscIO

Source/Core/DiscIO/VolumeWii.cpp

@@ -362,6 +362,33 @@ const BlobReader& VolumeWii::GetBlobReader() const
   return *m_reader;
 }
 
+std::array<u8, 20> VolumeWii::GetSyncHash() const
+{
+  mbedtls_sha1_context context;
+  mbedtls_sha1_init(&context);
+  mbedtls_sha1_starts_ret(&context);
+
+  // Disc header
+  ReadAndAddToSyncHash(&context, 0, 0x80, PARTITION_NONE);
+
+  // Region code
+  ReadAndAddToSyncHash(&context, 0x4E000, 4, PARTITION_NONE);
+
+  // The data offset of the game partition - an important factor for disc drive timings
+  const u64 data_offset = PartitionOffsetToRawOffset(0, GetGamePartition());
+  mbedtls_sha1_update_ret(&context, reinterpret_cast<const u8*>(&data_offset), sizeof(data_offset));
+
+  // TMD
+  AddTMDToSyncHash(&context, GetGamePartition());
+
+  // Game partition contents
+  AddGamePartitionToSyncHash(&context);
+
+  std::array<u8, 20> hash;
+  mbedtls_sha1_finish_ret(&context, hash.data());
+  return hash;
+}
+
 bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const
 {
   auto it = m_partitions.find(partition);

Source/Core/DiscIO/VolumeWii.h

@@ -92,6 +92,7 @@ public:
   bool IsSizeAccurate() const override;
   u64 GetRawSize() const override;
   const BlobReader& GetBlobReader() const override;
+  std::array<u8, 20> GetSyncHash() const override;
 
   // The in parameter can either contain all the data to begin with,
   // or read_function can write data into the in parameter when called.