From 7f57c24172857d24c8b7e2889f793fb3bb3a744a Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Tue, 21 Sep 2021 07:40:59 +0200 Subject: [PATCH 01/29] DiscIO/DirectoryBlob: Rename the vector-reference variant of DiscContentContainer::Add() to AddReference() to make it clearer at call sites that the given vector must remain in memory. --- Source/Core/DiscIO/DirectoryBlob.cpp | 18 ++++++++++-------- Source/Core/DiscIO/DirectoryBlob.h | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index c8352c79e1..778adf7e3c 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -489,7 +489,8 @@ void DirectoryBlobReader::SetNonpartitionDiscHeader(const std::vector& parti m_encrypted = std::all_of(m_disc_header_nonpartition.data() + 0x60, m_disc_header_nonpartition.data() + 0x64, [](u8 x) { return x == 0; }); - m_nonpartition_contents.Add(WII_NONPARTITION_DISCHEADER_ADDRESS, m_disc_header_nonpartition); + m_nonpartition_contents.AddReference(WII_NONPARTITION_DISCHEADER_ADDRESS, + m_disc_header_nonpartition); } void DirectoryBlobReader::SetWiiRegionData(const std::string& game_partition_root) @@ -505,7 +506,7 @@ void DirectoryBlobReader::SetWiiRegionData(const std::string& game_partition_roo else if (bytes_read < 0x20) ERROR_LOG_FMT(DISCIO, "Couldn't read age ratings from {}", region_bin_path); - m_nonpartition_contents.Add(WII_REGION_DATA_ADDRESS, m_wii_region_data); + m_nonpartition_contents.AddReference(WII_REGION_DATA_ADDRESS, m_wii_region_data); } void DirectoryBlobReader::SetPartitions(std::vector&& partitions) @@ -571,7 +572,7 @@ void DirectoryBlobReader::SetPartitions(std::vector&& partiti } m_data_size = partition_address; - m_nonpartition_contents.Add(PARTITION_TABLE_ADDRESS, m_partition_table); + m_nonpartition_contents.AddReference(PARTITION_TABLE_ADDRESS, m_partition_table); } // This function sets the header that's shortly before the start of the encrypted @@ -611,7 +612,8 @@ void DirectoryBlobReader::SetPartitionHeader(DirectoryBlobPartition* partition, Write32(PARTITION_DATA_OFFSET >> 2, 0x14, &partition_header); Write32(static_cast(data_size >> 2), 0x18, &partition_header); - m_nonpartition_contents.Add(partition_address + WII_PARTITION_TICKET_SIZE, partition_header); + m_nonpartition_contents.AddReference(partition_address + WII_PARTITION_TICKET_SIZE, + partition_header); std::vector ticket_buffer(ticket_size); m_nonpartition_contents.Read(partition_address + WII_PARTITION_TICKET_ADDRESS, ticket_size, @@ -637,7 +639,7 @@ void DirectoryBlobPartition::SetDiscHeaderAndDiscType(std::optional is_wii if (ReadFileToVector(boot_bin_path, &m_disc_header) < 0x20) ERROR_LOG_FMT(DISCIO, "{} doesn't exist or is too small", boot_bin_path); - m_contents.Add(DISCHEADER_ADDRESS, m_disc_header); + m_contents.AddReference(DISCHEADER_ADDRESS, m_disc_header); if (is_wii.has_value()) { @@ -666,7 +668,7 @@ void DirectoryBlobPartition::SetBI2() if (!m_is_wii && bytes_read < 0x1C) ERROR_LOG_FMT(DISCIO, "Couldn't read region from {}", bi2_path); - m_contents.Add(BI2_ADDRESS, m_bi2); + m_contents.AddReference(BI2_ADDRESS, m_bi2); } u64 DirectoryBlobPartition::SetApploader() @@ -697,7 +699,7 @@ u64 DirectoryBlobPartition::SetApploader() Write32(static_cast(-1), 0x10, &m_apploader); } - m_contents.Add(APPLOADER_ADDRESS, m_apploader); + m_contents.AddReference(APPLOADER_ADDRESS, m_apploader); // Return DOL address, 32 byte aligned (plus 32 byte padding) return Common::AlignUp(APPLOADER_ADDRESS + m_apploader.size() + 0x20, 0x20ull); @@ -748,7 +750,7 @@ void DirectoryBlobPartition::BuildFST(u64 fst_address) Write32((u32)(m_fst_data.size() >> m_address_shift), 0x0428, &m_disc_header); Write32((u32)(m_fst_data.size() >> m_address_shift), 0x042c, &m_disc_header); - m_contents.Add(fst_address, m_fst_data); + m_contents.AddReference(fst_address, m_fst_data); m_data_size = current_data_address; } diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index e4eb1b3cd3..5a27791477 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -71,7 +71,7 @@ class DiscContentContainer { public: template - void Add(u64 offset, const std::vector& vector) + void AddReference(u64 offset, const std::vector& vector) { return Add(offset, vector.size() * sizeof(T), reinterpret_cast(vector.data())); } From b988ab44419560e511e587e2473b0f259d434f83 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Tue, 21 Sep 2021 07:58:41 +0200 Subject: [PATCH 02/29] DiscIO/DirectoryBlob: Consolidate functions that take data for ContentSource to just take a ContentSource. --- Source/Core/DiscIO/DirectoryBlob.cpp | 30 ++++------------------------ Source/Core/DiscIO/DirectoryBlob.h | 20 ++++++++----------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index 778adf7e3c..f731e9b13f 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -61,18 +61,8 @@ constexpr u8 ENTRY_SIZE = 0x0c; constexpr u8 FILE_ENTRY = 0; constexpr u8 DIRECTORY_ENTRY = 1; -DiscContent::DiscContent(u64 offset, u64 size, const std::string& path) - : m_offset(offset), m_size(size), m_content_source(path) -{ -} - -DiscContent::DiscContent(u64 offset, u64 size, const u8* data) - : m_offset(offset), m_size(size), m_content_source(data) -{ -} - -DiscContent::DiscContent(u64 offset, u64 size, DirectoryBlobReader* blob) - : m_offset(offset), m_size(size), m_content_source(blob) +DiscContent::DiscContent(u64 offset, u64 size, ContentSource source) + : m_offset(offset), m_size(size), m_content_source(std::move(source)) { } @@ -137,22 +127,10 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer) const return true; } -void DiscContentContainer::Add(u64 offset, u64 size, const std::string& path) +void DiscContentContainer::Add(u64 offset, u64 size, ContentSource source) { if (size != 0) - m_contents.emplace(offset, size, path); -} - -void DiscContentContainer::Add(u64 offset, u64 size, const u8* data) -{ - if (size != 0) - m_contents.emplace(offset, size, data); -} - -void DiscContentContainer::Add(u64 offset, u64 size, DirectoryBlobReader* blob) -{ - if (size != 0) - m_contents.emplace(offset, size, blob); + m_contents.emplace(offset, size, std::move(source)); } u64 DiscContentContainer::CheckSizeAndAdd(u64 offset, const std::string& path) diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 5a27791477..4281c7159c 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -33,18 +33,16 @@ class DirectoryBlobReader; // Returns true if the path is inside a DirectoryBlob and doesn't represent the DirectoryBlob itself bool ShouldHideFromGameList(const std::string& volume_path); +using ContentSource = + std::variant; + class DiscContent { public: - using ContentSource = - std::variant; - - DiscContent(u64 offset, u64 size, const std::string& path); - DiscContent(u64 offset, u64 size, const u8* data); - DiscContent(u64 offset, u64 size, DirectoryBlobReader* blob); + DiscContent(u64 offset, u64 size, ContentSource source); // Provided because it's convenient when searching for DiscContent in an std::set explicit DiscContent(u64 offset); @@ -75,9 +73,7 @@ public: { return Add(offset, vector.size() * sizeof(T), reinterpret_cast(vector.data())); } - void Add(u64 offset, u64 size, const std::string& path); - void Add(u64 offset, u64 size, const u8* data); - void Add(u64 offset, u64 size, DirectoryBlobReader* blob); + void Add(u64 offset, u64 size, ContentSource source); u64 CheckSizeAndAdd(u64 offset, const std::string& path); u64 CheckSizeAndAdd(u64 offset, u64 max_size, const std::string& path); From 885e6690c58734aed5f8ce242f212149b059078f Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 01:30:08 +0200 Subject: [PATCH 03/29] DiscIO/DirectoryBlob: Add explanations for DiscContent members. --- Source/Core/DiscIO/DirectoryBlob.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 4281c7159c..93679045dc 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -60,8 +60,13 @@ public: bool operator>=(const DiscContent& other) const { return !(*this < other); } private: + // Position of this content chunk within its parent DiscContentContainer. u64 m_offset; + + // Number of bytes this content chunk takes up. u64 m_size = 0; + + // Where and how to find the data for this content chunk. ContentSource m_content_source; }; From a14436fe3fea883a27337c2f75bb809fcc076ec7 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 01:37:45 +0200 Subject: [PATCH 04/29] DiscIO/DirectoryBlob: Add ability to have a start-of-file offset for a file ContentSource. --- Source/Core/DiscIO/DirectoryBlob.cpp | 16 ++++++++++------ Source/Core/DiscIO/DirectoryBlob.h | 12 +++++++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index f731e9b13f..0396e7ecf2 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -97,11 +97,15 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer) const { const u64 bytes_to_read = std::min(m_size - offset_in_content, *length); - if (std::holds_alternative(m_content_source)) + if (std::holds_alternative(m_content_source)) { - File::IOFile file(std::get(m_content_source), "rb"); - if (!file.Seek(offset_in_content, SEEK_SET) || !file.ReadBytes(*buffer, bytes_to_read)) + const auto& content = std::get(m_content_source); + File::IOFile file(content.m_filename, "rb"); + if (!file.Seek(content.m_offset + offset_in_content, SEEK_SET) || + !file.ReadBytes(*buffer, bytes_to_read)) + { return false; + } } else if (std::holds_alternative(m_content_source)) { @@ -136,14 +140,14 @@ void DiscContentContainer::Add(u64 offset, u64 size, ContentSource source) u64 DiscContentContainer::CheckSizeAndAdd(u64 offset, const std::string& path) { const u64 size = File::GetSize(path); - Add(offset, size, path); + Add(offset, size, ContentFile{path, 0}); return size; } u64 DiscContentContainer::CheckSizeAndAdd(u64 offset, u64 max_size, const std::string& path) { const u64 size = std::min(File::GetSize(path), max_size); - Add(offset, size, path); + Add(offset, size, ContentFile{path, 0}); return size; } @@ -791,7 +795,7 @@ void DirectoryBlobPartition::WriteDirectory(const File::FSTEntry& parent_entry, WriteEntryName(name_offset, entry.virtualName, name_table_offset); // write entry to virtual disc - m_contents.Add(*data_offset, entry.size, entry.physicalName); + m_contents.Add(*data_offset, entry.size, ContentFile{entry.physicalName, 0}); // 32 KiB aligned - many games are fine with less alignment, but not all *data_offset = Common::AlignUp(*data_offset + entry.size, 0x8000ull); diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 93679045dc..ff7c8e6182 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -33,8 +33,18 @@ class DirectoryBlobReader; // Returns true if the path is inside a DirectoryBlob and doesn't represent the DirectoryBlob itself bool ShouldHideFromGameList(const std::string& volume_path); +// Content chunk that is loaded from a file in the host file system. +struct ContentFile +{ + // Path where the file can be found. + std::string m_filename; + + // Offset from the start of the file where the first byte of this content chunk is. + u64 m_offset; +}; + using ContentSource = - std::variant; From b7a9cc37b1872dbd8e5f2ca8e3f715e546798433 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 02:51:10 +0200 Subject: [PATCH 05/29] DiscIO/DirectoryBlob: Add ability to have an offset for a partition ContentSource. --- Source/Core/DiscIO/DirectoryBlob.cpp | 21 ++++++++++++++------- Source/Core/DiscIO/DirectoryBlob.h | 23 ++++++++++++++++++----- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index 0396e7ecf2..e76fa7ad27 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -112,16 +112,22 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer) const const u8* const content_pointer = std::get(m_content_source) + offset_in_content; std::copy(content_pointer, content_pointer + bytes_to_read, *buffer); } - else + else if (std::holds_alternative(m_content_source)) { - DirectoryBlobReader* blob = std::get(m_content_source); + const auto& content = std::get(m_content_source); + DirectoryBlobReader* blob = content.m_reader; const u64 decrypted_size = m_size * VolumeWii::BLOCK_DATA_SIZE / VolumeWii::BLOCK_TOTAL_SIZE; - if (!blob->EncryptPartitionData(offset_in_content, bytes_to_read, *buffer, m_offset, - decrypted_size)) + if (!blob->EncryptPartitionData(content.m_offset + offset_in_content, bytes_to_read, *buffer, + content.m_partition_data_offset, decrypted_size)) { return false; } } + else + { + PanicAlertFmt("DirectoryBlob: Invalid content source in DiscContent."); + return false; + } *length -= bytes_to_read; *buffer += bytes_to_read; @@ -545,9 +551,10 @@ void DirectoryBlobReader::SetPartitions(std::vector&& partiti SetPartitionHeader(&partitions[i].partition, partition_address); const u64 data_size = partitions[i].partition.GetDataSize(); - m_partitions.emplace(partition_address + PARTITION_DATA_OFFSET, - std::move(partitions[i].partition)); - m_nonpartition_contents.Add(partition_address + PARTITION_DATA_OFFSET, data_size, this); + const u64 partition_data_offset = partition_address + PARTITION_DATA_OFFSET; + m_partitions.emplace(partition_data_offset, std::move(partitions[i].partition)); + m_nonpartition_contents.Add(partition_data_offset, data_size, + ContentPartition{this, 0, partition_data_offset}); const u64 unaligned_next_partition_address = VolumeWii::EncryptedPartitionOffsetToRawOffset( data_size, Partition(partition_address), PARTITION_DATA_OFFSET); partition_address = Common::AlignUp(unaligned_next_partition_address, 0x10000ull); diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index ff7c8e6182..e542705f9f 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -43,11 +43,24 @@ struct ContentFile u64 m_offset; }; -using ContentSource = - std::variant; +// Content chunk that loads data from a DirectoryBlobReader. +// Intented for representing a partition within a disc. +struct ContentPartition +{ + // The reader to read data from. + DirectoryBlobReader* m_reader; + + // Offset from the start of the partition for the first byte represented by this chunk. + u64 m_offset; + + // The value passed as partition_data_offset to EncryptPartitionData(). + u64 m_partition_data_offset; +}; + +using ContentSource = std::variant; class DiscContent { From f8611f71395f0dc469582c8cdfce7c601c635b00 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 02:57:31 +0200 Subject: [PATCH 06/29] DiscIO/DirectoryBlob: Add a content source that reads data from a DiscIO::Volume. --- Source/Core/DiscIO/DirectoryBlob.cpp | 9 +++++++++ Source/Core/DiscIO/DirectoryBlob.h | 21 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index e76fa7ad27..c86b0396a2 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -123,6 +123,15 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer) const return false; } } + else if (std::holds_alternative(m_content_source)) + { + const auto& source = std::get(m_content_source); + if (!source.m_volume->Read(source.m_offset + offset_in_content, bytes_to_read, *buffer, + source.m_partition)) + { + return false; + } + } else { PanicAlertFmt("DirectoryBlob: Invalid content source in DiscContent."); diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index e542705f9f..e942c14a94 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -16,6 +16,7 @@ #include "Common/CommonTypes.h" #include "Common/FileUtil.h" #include "DiscIO/Blob.h" +#include "DiscIO/Volume.h" #include "DiscIO/WiiEncryptionCache.h" namespace File @@ -57,9 +58,23 @@ struct ContentPartition u64 m_partition_data_offset; }; -using ContentSource = std::variant; class DiscContent From b1802f6daa65e8617b26c4b47d94813242fb7945 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 03:02:39 +0200 Subject: [PATCH 07/29] DiscIO/DirectoryBlob: Add a content source representing a run of padding bytes. --- Source/Core/DiscIO/DirectoryBlob.cpp | 5 +++++ Source/Core/DiscIO/DirectoryBlob.h | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index c86b0396a2..356416b076 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -132,6 +132,11 @@ bool DiscContent::Read(u64* offset, u64* length, u8** buffer) const return false; } } + else if (std::holds_alternative(m_content_source)) + { + const ContentFixedByte& source = std::get(m_content_source); + std::fill_n(*buffer, bytes_to_read, source.m_byte); + } else { PanicAlertFmt("DirectoryBlob: Invalid content source in DiscContent."); diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index e942c14a94..5ab03137e5 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -71,10 +71,18 @@ struct ContentVolume Partition m_partition; }; +// Content chunk representing a run of identical bytes. +// Useful for padding between chunks within a file. +struct ContentFixedByte +{ + u8 m_byte; +}; + using ContentSource = std::variant; class DiscContent From 3a72a39efd0721789e09e25dc6c9f06c9515fbce Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 04:18:05 +0200 Subject: [PATCH 08/29] DiscIO/DirectoryBlob: Allow constructing a DirectoryBlobPartition from a VolumeDisc. --- Source/Core/DiscIO/DirectoryBlob.cpp | 296 +++++++++++++++++++++------ Source/Core/DiscIO/DirectoryBlob.h | 71 ++++++- 2 files changed, 295 insertions(+), 72 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index 356416b076..06a6bf1b8c 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -28,6 +28,7 @@ #include "Core/IOS/ES/Formats.h" #include "DiscIO/Blob.h" #include "DiscIO/DiscUtils.h" +#include "DiscIO/VolumeDisc.h" #include "DiscIO/VolumeWii.h" #include "DiscIO/WiiEncryptionCache.h" @@ -40,9 +41,7 @@ static size_t ReadFileToVector(const std::string& path, std::vector* vector) static void PadToAddress(u64 start_address, u64* address, u64* length, u8** buffer); static void Write32(u32 data, u32 offset, std::vector* buffer); -static u32 ComputeNameSize(const File::FSTEntry& parent_entry); static std::string ASCIIToUppercase(std::string str); -static void ConvertUTF8NamesToSHIFTJIS(File::FSTEntry* parent_entry); enum class PartitionType : u32 { @@ -626,24 +625,112 @@ void DirectoryBlobReader::SetPartitionHeader(DirectoryBlobPartition* partition, partition->SetKey(ticket.GetTitleKey()); } +static void GenerateBuilderNodesFromFileSystem(const DiscIO::VolumeDisc& volume, + const DiscIO::Partition& partition, + std::vector* nodes, + const FileInfo& parent_info) +{ + for (const FileInfo& file_info : parent_info) + { + if (file_info.IsDirectory()) + { + std::vector child_nodes; + GenerateBuilderNodesFromFileSystem(volume, partition, &child_nodes, file_info); + nodes->emplace_back(FSTBuilderNode{file_info.GetName(), file_info.GetTotalChildren(), + std::move(child_nodes)}); + } + else + { + std::vector source; + source.emplace_back(BuilderContentSource{ + 0, file_info.GetSize(), ContentVolume{file_info.GetOffset(), &volume, partition}}); + nodes->emplace_back( + FSTBuilderNode{file_info.GetName(), file_info.GetSize(), std::move(source)}); + } + } +} + DirectoryBlobPartition::DirectoryBlobPartition(const std::string& root_directory, std::optional is_wii) : m_root_directory(root_directory) { - SetDiscHeaderAndDiscType(is_wii); - SetBI2(); - BuildFST(SetDOL(SetApploader())); + SetDiscHeaderFromFile(m_root_directory + "sys/boot.bin"); + SetDiscType(is_wii); + SetBI2FromFile(m_root_directory + "sys/bi2.bin"); + const u64 dol_address = SetApploaderFromFile(m_root_directory + "sys/apploader.img"); + const u64 fst_address = SetDOLFromFile(m_root_directory + "sys/main.dol", dol_address); + BuildFSTFromFolder(m_root_directory + "files/", fst_address); } -void DirectoryBlobPartition::SetDiscHeaderAndDiscType(std::optional is_wii) +DirectoryBlobPartition::DirectoryBlobPartition(DiscIO::VolumeDisc* volume, + const DiscIO::Partition& partition, + std::optional is_wii) + : m_wrapped_partition(partition) +{ + std::vector disc_header(DISCHEADER_SIZE); + if (!volume->Read(DISCHEADER_ADDRESS, DISCHEADER_SIZE, disc_header.data(), partition)) + disc_header.clear(); + SetDiscHeader(std::move(disc_header)); + SetDiscType(is_wii); + + std::vector bi2(BI2_SIZE); + if (!volume->Read(BI2_ADDRESS, BI2_SIZE, bi2.data(), partition)) + bi2.clear(); + SetBI2(std::move(bi2)); + + std::vector apploader; + const auto apploader_size = GetApploaderSize(*volume, partition); + if (apploader_size) + { + apploader.resize(*apploader_size); + if (!volume->Read(APPLOADER_ADDRESS, *apploader_size, apploader.data(), partition)) + apploader.clear(); + } + const u64 new_dol_address = SetApploader(apploader, "apploader"); + + FSTBuilderNode dol_node{"main.dol", 0, {}}; + const auto dol_offset = GetBootDOLOffset(*volume, partition); + if (dol_offset) + { + const auto dol_size = GetBootDOLSize(*volume, partition, *dol_offset); + if (dol_size) + { + std::vector dol_contents; + dol_contents.emplace_back( + BuilderContentSource{0, *dol_size, ContentVolume{*dol_offset, volume, partition}}); + dol_node.m_size = *dol_size; + dol_node.m_content = std::move(dol_contents); + } + } + const u64 new_fst_address = SetDOL(std::move(dol_node), new_dol_address); + + const FileSystem* fs = volume->GetFileSystem(partition); + if (!fs || !fs->IsValid()) + return; + + std::vector nodes; + GenerateBuilderNodesFromFileSystem(*volume, partition, &nodes, fs->GetRoot()); + BuildFST(std::move(nodes), new_fst_address); +} + +void DirectoryBlobPartition::SetDiscHeaderFromFile(const std::string& boot_bin_path) { m_disc_header.resize(DISCHEADER_SIZE); - const std::string boot_bin_path = m_root_directory + "sys/boot.bin"; if (ReadFileToVector(boot_bin_path, &m_disc_header) < 0x20) ERROR_LOG_FMT(DISCIO, "{} doesn't exist or is too small", boot_bin_path); m_contents.AddReference(DISCHEADER_ADDRESS, m_disc_header); +} +void DirectoryBlobPartition::SetDiscHeader(std::vector boot_bin) +{ + m_disc_header = std::move(boot_bin); + m_disc_header.resize(DISCHEADER_SIZE); + m_contents.AddReference(DISCHEADER_ADDRESS, m_disc_header); +} + +void DirectoryBlobPartition::SetDiscType(std::optional is_wii) +{ if (is_wii.has_value()) { m_is_wii = *is_wii; @@ -653,20 +740,22 @@ void DirectoryBlobPartition::SetDiscHeaderAndDiscType(std::optional is_wii m_is_wii = Common::swap32(&m_disc_header[0x18]) == WII_DISC_MAGIC; const bool is_gc = Common::swap32(&m_disc_header[0x1c]) == GAMECUBE_DISC_MAGIC; if (m_is_wii == is_gc) - ERROR_LOG_FMT(DISCIO, "Couldn't detect disc type based on {}", boot_bin_path); + { + ERROR_LOG_FMT(DISCIO, "Couldn't detect disc type based on disc header; assuming {}", + m_is_wii ? "Wii" : "GameCube"); + } } m_address_shift = m_is_wii ? 2 : 0; } -void DirectoryBlobPartition::SetBI2() +void DirectoryBlobPartition::SetBI2FromFile(const std::string& bi2_path) { m_bi2.resize(BI2_SIZE); if (!m_is_wii) Write32(INVALID_REGION, 0x18, &m_bi2); - const std::string bi2_path = m_root_directory + "sys/bi2.bin"; const size_t bytes_read = ReadFileToVector(bi2_path, &m_bi2); if (!m_is_wii && bytes_read < 0x1C) ERROR_LOG_FMT(DISCIO, "Couldn't read region from {}", bi2_path); @@ -674,23 +763,41 @@ void DirectoryBlobPartition::SetBI2() m_contents.AddReference(BI2_ADDRESS, m_bi2); } -u64 DirectoryBlobPartition::SetApploader() +void DirectoryBlobPartition::SetBI2(std::vector bi2) +{ + const size_t bi2_size = bi2.size(); + m_bi2 = std::move(bi2); + m_bi2.resize(BI2_SIZE); + + if (!m_is_wii && bi2_size < 0x1C) + Write32(INVALID_REGION, 0x18, &m_bi2); + + m_contents.AddReference(BI2_ADDRESS, m_bi2); +} + +u64 DirectoryBlobPartition::SetApploaderFromFile(const std::string& path) +{ + File::IOFile file(path, "rb"); + std::vector apploader(file.GetSize()); + file.ReadBytes(apploader.data(), apploader.size()); + return SetApploader(std::move(apploader), path); +} + +u64 DirectoryBlobPartition::SetApploader(std::vector apploader, const std::string& log_path) { bool success = false; - const std::string path = m_root_directory + "sys/apploader.img"; - File::IOFile file(path, "rb"); - m_apploader.resize(file.GetSize()); - if (m_apploader.size() < 0x20 || !file.ReadBytes(m_apploader.data(), m_apploader.size())) + m_apploader = std::move(apploader); + if (m_apploader.size() < 0x20) { - ERROR_LOG_FMT(DISCIO, "{} couldn't be accessed or is too small", path); + ERROR_LOG_FMT(DISCIO, "{} couldn't be accessed or is too small", log_path); } else { const size_t apploader_size = 0x20 + Common::swap32(*(u32*)&m_apploader[0x14]) + Common::swap32(*(u32*)&m_apploader[0x18]); if (apploader_size != m_apploader.size()) - ERROR_LOG_FMT(DISCIO, "{} is the wrong size... Is it really an apploader?", path); + ERROR_LOG_FMT(DISCIO, "{} is the wrong size... Is it really an apploader?", log_path); else success = true; } @@ -708,9 +815,9 @@ u64 DirectoryBlobPartition::SetApploader() return Common::AlignUp(APPLOADER_ADDRESS + m_apploader.size() + 0x20, 0x20ull); } -u64 DirectoryBlobPartition::SetDOL(u64 dol_address) +u64 DirectoryBlobPartition::SetDOLFromFile(const std::string& path, u64 dol_address) { - const u64 dol_size = m_contents.CheckSizeAndAdd(dol_address, m_root_directory + "sys/main.dol"); + const u64 dol_size = m_contents.CheckSizeAndAdd(dol_address, path); Write32(static_cast(dol_address >> m_address_shift), 0x0420, &m_disc_header); @@ -718,16 +825,92 @@ u64 DirectoryBlobPartition::SetDOL(u64 dol_address) return Common::AlignUp(dol_address + dol_size + 0x20, 0x20ull); } -void DirectoryBlobPartition::BuildFST(u64 fst_address) +u64 DirectoryBlobPartition::SetDOL(FSTBuilderNode dol_node, u64 dol_address) +{ + for (auto& content : dol_node.GetFileContent()) + m_contents.Add(dol_address + content.m_offset, content.m_size, std::move(content.m_source)); + + Write32(static_cast(dol_address >> m_address_shift), 0x0420, &m_disc_header); + + // Return FST address, 32 byte aligned (plus 32 byte padding) + return Common::AlignUp(dol_address + dol_node.m_size + 0x20, 0x20ull); +} + +static std::vector ConvertFSTEntriesToBuilderNodes(const File::FSTEntry& parent) +{ + std::vector nodes; + nodes.reserve(parent.children.size()); + for (const File::FSTEntry& entry : parent.children) + { + std::variant, std::vector> content; + if (entry.isDirectory) + { + content = ConvertFSTEntriesToBuilderNodes(entry); + } + else + { + content = + std::vector{{0, entry.size, ContentFile{entry.physicalName, 0}}}; + } + + nodes.emplace_back(FSTBuilderNode{entry.virtualName, entry.size, std::move(content)}); + } + return nodes; +} + +void DirectoryBlobPartition::BuildFSTFromFolder(const std::string& fst_root_path, u64 fst_address) +{ + auto nodes = ConvertFSTEntriesToBuilderNodes(File::ScanDirectoryTree(fst_root_path, true)); + BuildFST(std::move(nodes), fst_address); +} + +static void ConvertUTF8NamesToSHIFTJIS(std::vector* fst) +{ + for (FSTBuilderNode& entry : *fst) + { + if (entry.IsFolder()) + ConvertUTF8NamesToSHIFTJIS(&entry.GetFolderContent()); + entry.m_filename = UTF8ToSHIFTJIS(entry.m_filename); + } +} + +static u32 ComputeNameSize(const std::vector& files) +{ + u32 name_size = 0; + for (const FSTBuilderNode& entry : files) + { + if (entry.IsFolder()) + name_size += ComputeNameSize(entry.GetFolderContent()); + name_size += static_cast(entry.m_filename.length() + 1); + } + return name_size; +} + +static size_t RecalculateFolderSizes(std::vector* fst) +{ + size_t size = 0; + for (FSTBuilderNode& entry : *fst) + { + ++size; + if (entry.IsFile()) + continue; + + entry.m_size = RecalculateFolderSizes(&entry.GetFolderContent()); + size += entry.m_size; + } + return size; +} + +void DirectoryBlobPartition::BuildFST(std::vector root_nodes, u64 fst_address) { m_fst_data.clear(); - File::FSTEntry rootEntry = File::ScanDirectoryTree(m_root_directory + "files/", true); + ConvertUTF8NamesToSHIFTJIS(&root_nodes); - ConvertUTF8NamesToSHIFTJIS(&rootEntry); + u32 name_table_size = Common::AlignUp(ComputeNameSize(root_nodes), 1ull << m_address_shift); - u32 name_table_size = Common::AlignUp(ComputeNameSize(rootEntry), 1ull << m_address_shift); - u64 total_entries = rootEntry.size + 1; // The root entry itself isn't counted in rootEntry.size + // 1 extra for the root entry + u64 total_entries = RecalculateFolderSizes(&root_nodes) + 1; const u64 name_table_offset = total_entries * ENTRY_SIZE; m_fst_data.resize(name_table_offset + name_table_size); @@ -742,7 +925,7 @@ void DirectoryBlobPartition::BuildFST(u64 fst_address) // write root entry WriteEntryData(&fst_offset, DIRECTORY_ENTRY, 0, 0, total_entries, m_address_shift); - WriteDirectory(rootEntry, &fst_offset, &name_offset, ¤t_data_address, root_offset, + WriteDirectory(&root_nodes, &fst_offset, &name_offset, ¤t_data_address, root_offset, name_table_offset); // overflow check, compare the aligned name offset with the aligned name table size @@ -782,44 +965,51 @@ void DirectoryBlobPartition::WriteEntryName(u32* name_offset, const std::string& *name_offset += (u32)(name.length() + 1); } -void DirectoryBlobPartition::WriteDirectory(const File::FSTEntry& parent_entry, u32* fst_offset, - u32* name_offset, u64* data_offset, +void DirectoryBlobPartition::WriteDirectory(std::vector* parent_entries, + u32* fst_offset, u32* name_offset, u64* data_offset, u32 parent_entry_index, u64 name_table_offset) { - std::vector sorted_entries = parent_entry.children; + std::vector& sorted_entries = *parent_entries; // Sort for determinism std::sort(sorted_entries.begin(), sorted_entries.end(), - [](const File::FSTEntry& one, const File::FSTEntry& two) { - const std::string one_upper = ASCIIToUppercase(one.virtualName); - const std::string two_upper = ASCIIToUppercase(two.virtualName); - return one_upper == two_upper ? one.virtualName < two.virtualName : + [](const FSTBuilderNode& one, const FSTBuilderNode& two) { + const std::string one_upper = ASCIIToUppercase(one.m_filename); + const std::string two_upper = ASCIIToUppercase(two.m_filename); + return one_upper == two_upper ? one.m_filename < two.m_filename : one_upper < two_upper; }); - for (const File::FSTEntry& entry : sorted_entries) + for (FSTBuilderNode& entry : sorted_entries) { - if (entry.isDirectory) + if (entry.IsFolder()) { u32 entry_index = *fst_offset / ENTRY_SIZE; WriteEntryData(fst_offset, DIRECTORY_ENTRY, *name_offset, parent_entry_index, - entry_index + entry.size + 1, 0); - WriteEntryName(name_offset, entry.virtualName, name_table_offset); + entry_index + entry.m_size + 1, 0); + WriteEntryName(name_offset, entry.m_filename, name_table_offset); - WriteDirectory(entry, fst_offset, name_offset, data_offset, entry_index, name_table_offset); + auto& child_nodes = entry.GetFolderContent(); + WriteDirectory(&child_nodes, fst_offset, name_offset, data_offset, entry_index, + name_table_offset); } else { // put entry in FST - WriteEntryData(fst_offset, FILE_ENTRY, *name_offset, *data_offset, entry.size, + WriteEntryData(fst_offset, FILE_ENTRY, *name_offset, *data_offset, entry.m_size, m_address_shift); - WriteEntryName(name_offset, entry.virtualName, name_table_offset); + WriteEntryName(name_offset, entry.m_filename, name_table_offset); // write entry to virtual disc - m_contents.Add(*data_offset, entry.size, ContentFile{entry.physicalName, 0}); + auto& contents = entry.GetFileContent(); + for (BuilderContentSource& content : contents) + { + m_contents.Add(*data_offset + content.m_offset, content.m_size, + std::move(content.m_source)); + } // 32 KiB aligned - many games are fine with less alignment, but not all - *data_offset = Common::AlignUp(*data_offset + entry.size, 0x8000ull); + *data_offset = Common::AlignUp(*data_offset + entry.m_size, 0x8000ull); } } } @@ -852,30 +1042,6 @@ static void Write32(u32 data, u32 offset, std::vector* buffer) (*buffer)[offset] = data & 0xff; } -static u32 ComputeNameSize(const File::FSTEntry& parent_entry) -{ - u32 name_size = 0; - for (const File::FSTEntry& entry : parent_entry.children) - { - if (entry.isDirectory) - name_size += ComputeNameSize(entry); - - name_size += (u32)entry.virtualName.length() + 1; - } - return name_size; -} - -static void ConvertUTF8NamesToSHIFTJIS(File::FSTEntry* parent_entry) -{ - for (File::FSTEntry& entry : parent_entry->children) - { - if (entry.isDirectory) - ConvertUTF8NamesToSHIFTJIS(&entry); - - entry.virtualName = UTF8ToSHIFTJIS(entry.virtualName); - } -} - static std::string ASCIIToUppercase(std::string str) { std::transform(str.begin(), str.end(), str.begin(), diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 5ab03137e5..cd274b8431 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -30,6 +30,7 @@ namespace DiscIO enum class PartitionType : u32; class DirectoryBlobReader; +class VolumeDisc; // Returns true if the path is inside a DirectoryBlob and doesn't represent the DirectoryBlob itself bool ShouldHideFromGameList(const std::string& volume_path); @@ -85,6 +86,47 @@ using ContentSource = std::variant; +struct BuilderContentSource +{ + u64 m_offset; + u64 m_size; + ContentSource m_source; +}; + +struct FSTBuilderNode +{ + std::string m_filename; + u64 m_size; + std::variant, std::vector> m_content; + + bool IsFile() const + { + return std::holds_alternative>(m_content); + } + + std::vector& GetFileContent() + { + return std::get>(m_content); + } + + const std::vector& GetFileContent() const + { + return std::get>(m_content); + } + + bool IsFolder() const { return std::holds_alternative>(m_content); } + + std::vector& GetFolderContent() + { + return std::get>(m_content); + } + + const std::vector& GetFolderContent() const + { + return std::get>(m_content); + } +}; + class DiscContent { public: @@ -139,6 +181,8 @@ class DirectoryBlobPartition public: DirectoryBlobPartition() = default; DirectoryBlobPartition(const std::string& root_directory, std::optional is_wii); + DirectoryBlobPartition(DiscIO::VolumeDisc* volume, const DiscIO::Partition& partition, + std::optional is_wii); // We do not allow copying, because it might mess up the pointers inside DiscContents DirectoryBlobPartition(const DirectoryBlobPartition&) = delete; @@ -151,27 +195,38 @@ public: const std::string& GetRootDirectory() const { return m_root_directory; } const std::vector& GetHeader() const { return m_disc_header; } const DiscContentContainer& GetContents() const { return m_contents; } + const std::optional& GetWrappedPartition() const + { + return m_wrapped_partition; + } const std::array& GetKey() const { return m_key; } void SetKey(std::array key) { m_key = key; } private: - void SetDiscHeaderAndDiscType(std::optional is_wii); - void SetBI2(); + void SetDiscHeaderFromFile(const std::string& boot_bin_path); + void SetDiscHeader(std::vector boot_bin); + void SetDiscType(std::optional is_wii); + void SetBI2FromFile(const std::string& bi2_path); + void SetBI2(std::vector bi2); // Returns DOL address - u64 SetApploader(); + u64 SetApploaderFromFile(const std::string& path); + u64 SetApploader(std::vector apploader, const std::string& log_path); // Returns FST address - u64 SetDOL(u64 dol_address); + u64 SetDOLFromFile(const std::string& path, u64 dol_address); + u64 SetDOL(FSTBuilderNode dol_node, u64 dol_address); - void BuildFST(u64 fst_address); + void BuildFSTFromFolder(const std::string& fst_root_path, u64 fst_address); + void BuildFST(std::vector root_nodes, u64 fst_address); // FST creation void WriteEntryData(u32* entry_offset, u8 type, u32 name_offset, u64 data_offset, u64 length, u32 address_shift); void WriteEntryName(u32* name_offset, const std::string& name, u64 name_table_offset); - void WriteDirectory(const File::FSTEntry& parent_entry, u32* fst_offset, u32* name_offset, - u64* data_offset, u32 parent_entry_index, u64 name_table_offset); + void WriteDirectory(std::vector* parent_entries, u32* fst_offset, + u32* name_offset, u64* data_offset, u32 parent_entry_index, + u64 name_table_offset); DiscContentContainer m_contents; std::vector m_disc_header; @@ -187,6 +242,8 @@ private: u32 m_address_shift = 0; u64 m_data_size = 0; + + std::optional m_wrapped_partition = std::nullopt; }; class DirectoryBlobReader : public BlobReader From 00ef9f2b4fc5b4817806e2f8a50d17030397f94b Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 04:35:27 +0200 Subject: [PATCH 09/29] DiscIO/DirectoryBlob: Allow constructing a DirectoryBlobReader from a VolumeDisc. --- Source/Core/DiscIO/DirectoryBlob.cpp | 207 +++++++++++++++++++++++---- Source/Core/DiscIO/DirectoryBlob.h | 13 +- 2 files changed, 187 insertions(+), 33 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index 06a6bf1b8c..b41a00cf66 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -356,6 +356,15 @@ std::unique_ptr DirectoryBlobReader::Create(const std::stri return std::unique_ptr(new DirectoryBlobReader(partition_root, true_root)); } +std::unique_ptr +DirectoryBlobReader::Create(std::unique_ptr volume) +{ + if (!volume) + return nullptr; + + return std::unique_ptr(new DirectoryBlobReader(std::move(volume))); +} + DirectoryBlobReader::DirectoryBlobReader(const std::string& game_partition_root, const std::string& true_root) : m_encryption_cache(this) @@ -371,8 +380,8 @@ DirectoryBlobReader::DirectoryBlobReader(const std::string& game_partition_root, } else { - SetNonpartitionDiscHeader(game_partition.GetHeader(), game_partition_root); - SetWiiRegionData(game_partition_root); + SetNonpartitionDiscHeaderFromFile(game_partition.GetHeader(), game_partition_root); + SetWiiRegionDataFromFile(game_partition_root); std::vector partitions; partitions.emplace_back(std::move(game_partition), PartitionType::Game); @@ -400,6 +409,58 @@ DirectoryBlobReader::DirectoryBlobReader(const std::string& game_partition_root, } } +DirectoryBlobReader::DirectoryBlobReader(std::unique_ptr volume) + : m_encryption_cache(this), m_wrapped_volume(std::move(volume)) +{ + DirectoryBlobPartition game_partition(m_wrapped_volume.get(), + m_wrapped_volume->GetGamePartition(), std::nullopt); + m_is_wii = game_partition.IsWii(); + + if (!m_is_wii) + { + m_gamecube_pseudopartition = std::move(game_partition); + m_data_size = m_gamecube_pseudopartition.GetDataSize(); + m_encrypted = false; + } + else + { + std::vector header_bin(WII_NONPARTITION_DISCHEADER_SIZE); + if (!m_wrapped_volume->Read(WII_NONPARTITION_DISCHEADER_ADDRESS, + WII_NONPARTITION_DISCHEADER_SIZE, header_bin.data(), + PARTITION_NONE)) + { + header_bin.clear(); + } + SetNonpartitionDiscHeader(game_partition.GetHeader(), std::move(header_bin)); + + std::vector wii_region_data(WII_REGION_DATA_SIZE); + if (!m_wrapped_volume->Read(WII_REGION_DATA_ADDRESS, WII_REGION_DATA_SIZE, + wii_region_data.data(), PARTITION_NONE)) + { + wii_region_data.clear(); + } + SetWiiRegionData(wii_region_data, "volume"); + + std::vector partitions; + partitions.emplace_back(std::move(game_partition), PartitionType::Game); + + for (Partition partition : m_wrapped_volume->GetPartitions()) + { + if (partition == m_wrapped_volume->GetGamePartition()) + continue; + + auto type = m_wrapped_volume->GetPartitionType(partition); + if (type) + { + partitions.emplace_back(DirectoryBlobPartition(m_wrapped_volume.get(), partition, m_is_wii), + static_cast(*type)); + } + } + + SetPartitions(std::move(partitions)); + } +} + bool DirectoryBlobReader::Read(u64 offset, u64 length, u8* buffer) { if (offset + length > m_data_size) @@ -469,22 +530,35 @@ u64 DirectoryBlobReader::GetDataSize() const return m_data_size; } -void DirectoryBlobReader::SetNonpartitionDiscHeader(const std::vector& partition_header, - const std::string& game_partition_root) +void DirectoryBlobReader::SetNonpartitionDiscHeaderFromFile(const std::vector& partition_header, + const std::string& game_partition_root) { - m_disc_header_nonpartition.resize(WII_NONPARTITION_DISCHEADER_SIZE); + std::vector header_bin(WII_NONPARTITION_DISCHEADER_SIZE); const size_t header_bin_bytes_read = - ReadFileToVector(game_partition_root + "disc/header.bin", &m_disc_header_nonpartition); + ReadFileToVector(game_partition_root + "disc/header.bin", &header_bin); + header_bin.resize(header_bin_bytes_read); + SetNonpartitionDiscHeader(partition_header, std::move(header_bin)); +} + +void DirectoryBlobReader::SetNonpartitionDiscHeader(const std::vector& partition_header, + std::vector header_bin) +{ + const size_t header_bin_size = header_bin.size(); + m_disc_header_nonpartition = std::move(header_bin); + m_disc_header_nonpartition.resize(WII_NONPARTITION_DISCHEADER_SIZE); // If header.bin is missing or smaller than expected, use the content of sys/boot.bin instead - std::copy(partition_header.data() + header_bin_bytes_read, - partition_header.data() + m_disc_header_nonpartition.size(), - m_disc_header_nonpartition.data() + header_bin_bytes_read); + if (header_bin_size < m_disc_header_nonpartition.size()) + { + std::copy(partition_header.data() + header_bin_size, + partition_header.data() + m_disc_header_nonpartition.size(), + m_disc_header_nonpartition.data() + header_bin_size); + } // 0x60 and 0x61 are the only differences between the partition and non-partition headers - if (header_bin_bytes_read < 0x60) + if (header_bin_size < 0x60) m_disc_header_nonpartition[0x60] = 0; - if (header_bin_bytes_read < 0x61) + if (header_bin_size < 0x61) m_disc_header_nonpartition[0x61] = 0; m_encrypted = std::all_of(m_disc_header_nonpartition.data() + 0x60, @@ -494,18 +568,30 @@ void DirectoryBlobReader::SetNonpartitionDiscHeader(const std::vector& parti m_disc_header_nonpartition); } -void DirectoryBlobReader::SetWiiRegionData(const std::string& game_partition_root) +void DirectoryBlobReader::SetWiiRegionDataFromFile(const std::string& game_partition_root) +{ + std::vector wii_region_data(WII_REGION_DATA_SIZE); + const std::string region_bin_path = game_partition_root + "disc/region.bin"; + const size_t bytes_read = ReadFileToVector(region_bin_path, &wii_region_data); + wii_region_data.resize(bytes_read); + SetWiiRegionData(wii_region_data, region_bin_path); +} + +void DirectoryBlobReader::SetWiiRegionData(const std::vector& wii_region_data, + const std::string& log_path) { m_wii_region_data.resize(0x10, 0x00); - m_wii_region_data.resize(0x20, 0x80); + m_wii_region_data.resize(WII_REGION_DATA_SIZE, 0x80); Write32(INVALID_REGION, 0, &m_wii_region_data); - const std::string region_bin_path = game_partition_root + "disc/region.bin"; - const size_t bytes_read = ReadFileToVector(region_bin_path, &m_wii_region_data); - if (bytes_read < 0x4) - ERROR_LOG_FMT(DISCIO, "Couldn't read region from {}", region_bin_path); - else if (bytes_read < 0x20) - ERROR_LOG_FMT(DISCIO, "Couldn't read age ratings from {}", region_bin_path); + std::copy_n(wii_region_data.begin(), + std::min(wii_region_data.size(), WII_REGION_DATA_SIZE), + m_wii_region_data.begin()); + + if (wii_region_data.size() < 0x4) + ERROR_LOG_FMT(DISCIO, "Couldn't read region from {}", log_path); + else if (wii_region_data.size() < 0x20) + ERROR_LOG_FMT(DISCIO, "Couldn't read age ratings from {}", log_path); m_nonpartition_contents.AddReference(WII_REGION_DATA_ADDRESS, m_wii_region_data); } @@ -585,27 +671,88 @@ void DirectoryBlobReader::SetPartitionHeader(DirectoryBlobPartition* partition, constexpr u32 TMD_OFFSET = 0x2c0; constexpr u32 H3_OFFSET = 0x4000; + const std::optional& wrapped_partition = partition->GetWrappedPartition(); const std::string& partition_root = partition->GetRootDirectory(); - const u64 ticket_size = m_nonpartition_contents.CheckSizeAndAdd( - partition_address + WII_PARTITION_TICKET_ADDRESS, WII_PARTITION_TICKET_SIZE, - partition_root + "ticket.bin"); + u64 ticket_size; + if (wrapped_partition) + { + const auto& ticket = m_wrapped_volume->GetTicket(*wrapped_partition).GetBytes(); + auto& new_ticket = m_extra_data.emplace_back(ticket); + if (new_ticket.size() > WII_PARTITION_TICKET_SIZE) + new_ticket.resize(WII_PARTITION_TICKET_SIZE); + ticket_size = new_ticket.size(); + m_nonpartition_contents.AddReference(partition_address + WII_PARTITION_TICKET_ADDRESS, + new_ticket); + } + else + { + ticket_size = m_nonpartition_contents.CheckSizeAndAdd( + partition_address + WII_PARTITION_TICKET_ADDRESS, WII_PARTITION_TICKET_SIZE, + partition_root + "ticket.bin"); + } - const u64 tmd_size = m_nonpartition_contents.CheckSizeAndAdd( - partition_address + TMD_OFFSET, IOS::ES::MAX_TMD_SIZE, partition_root + "tmd.bin"); + u64 tmd_size; + if (wrapped_partition) + { + const auto& tmd = m_wrapped_volume->GetTMD(*wrapped_partition).GetBytes(); + auto& new_tmd = m_extra_data.emplace_back(tmd); + if (new_tmd.size() > IOS::ES::MAX_TMD_SIZE) + new_tmd.resize(IOS::ES::MAX_TMD_SIZE); + tmd_size = new_tmd.size(); + m_nonpartition_contents.AddReference(partition_address + TMD_OFFSET, new_tmd); + } + else + { + tmd_size = m_nonpartition_contents.CheckSizeAndAdd( + partition_address + TMD_OFFSET, IOS::ES::MAX_TMD_SIZE, partition_root + "tmd.bin"); + } const u64 cert_offset = Common::AlignUp(TMD_OFFSET + tmd_size, 0x20ull); const u64 max_cert_size = H3_OFFSET - cert_offset; - const u64 cert_size = m_nonpartition_contents.CheckSizeAndAdd( - partition_address + cert_offset, max_cert_size, partition_root + "cert.bin"); - m_nonpartition_contents.CheckSizeAndAdd(partition_address + H3_OFFSET, WII_PARTITION_H3_SIZE, - partition_root + "h3.bin"); + u64 cert_size; + if (wrapped_partition) + { + const auto& cert = m_wrapped_volume->GetCertificateChain(*wrapped_partition); + auto& new_cert = m_extra_data.emplace_back(cert); + if (new_cert.size() > max_cert_size) + new_cert.resize(max_cert_size); + cert_size = new_cert.size(); + m_nonpartition_contents.AddReference(partition_address + cert_offset, new_cert); + } + else + { + cert_size = m_nonpartition_contents.CheckSizeAndAdd(partition_address + cert_offset, + max_cert_size, partition_root + "cert.bin"); + } + + if (wrapped_partition) + { + if (m_wrapped_volume->IsEncryptedAndHashed()) + { + const std::optional offset = m_wrapped_volume->ReadSwappedAndShifted( + wrapped_partition->offset + WII_PARTITION_H3_OFFSET_ADDRESS, PARTITION_NONE); + if (offset) + { + auto& new_h3 = m_extra_data.emplace_back(WII_PARTITION_H3_SIZE); + if (m_wrapped_volume->Read(wrapped_partition->offset + *offset, new_h3.size(), + new_h3.data(), PARTITION_NONE)) + { + m_nonpartition_contents.AddReference(partition_address + H3_OFFSET, new_h3); + } + } + } + } + else + { + m_nonpartition_contents.CheckSizeAndAdd(partition_address + H3_OFFSET, WII_PARTITION_H3_SIZE, + partition_root + "h3.bin"); + } constexpr u32 PARTITION_HEADER_SIZE = 0x1c; const u64 data_size = Common::AlignUp(partition->GetDataSize(), 0x7c00) / 0x7c00 * 0x8000; - m_partition_headers.emplace_back(PARTITION_HEADER_SIZE); - std::vector& partition_header = m_partition_headers.back(); + std::vector& partition_header = m_extra_data.emplace_back(PARTITION_HEADER_SIZE); Write32(static_cast(tmd_size), 0x0, &partition_header); Write32(TMD_OFFSET >> 2, 0x4, &partition_header); Write32(static_cast(cert_size), 0x8, &partition_header); diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index cd274b8431..db88367d8c 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -252,6 +252,7 @@ class DirectoryBlobReader : public BlobReader public: static std::unique_ptr Create(const std::string& dol_path); + static std::unique_ptr Create(std::unique_ptr volume); // We do not allow copying, because it might mess up the pointers inside DiscContents DirectoryBlobReader(const DirectoryBlobReader&) = delete; @@ -287,15 +288,19 @@ private: explicit DirectoryBlobReader(const std::string& game_partition_root, const std::string& true_root); + explicit DirectoryBlobReader(std::unique_ptr volume); const DirectoryBlobPartition* GetPartition(u64 offset, u64 size, u64 partition_data_offset) const; bool EncryptPartitionData(u64 offset, u64 size, u8* buffer, u64 partition_data_offset, u64 partition_data_decrypted_size); + void SetNonpartitionDiscHeaderFromFile(const std::vector& partition_header, + const std::string& game_partition_root); void SetNonpartitionDiscHeader(const std::vector& partition_header, - const std::string& game_partition_root); - void SetWiiRegionData(const std::string& game_partition_root); + std::vector header_bin); + void SetWiiRegionDataFromFile(const std::string& game_partition_root); + void SetWiiRegionData(const std::vector& wii_region_data, const std::string& log_path); void SetPartitions(std::vector&& partitions); void SetPartitionHeader(DirectoryBlobPartition* partition, u64 partition_address); @@ -313,9 +318,11 @@ private: std::vector m_disc_header_nonpartition; std::vector m_partition_table; std::vector m_wii_region_data; - std::vector> m_partition_headers; + std::vector> m_extra_data; u64 m_data_size; + + std::unique_ptr m_wrapped_volume; }; } // namespace DiscIO From e3f1de023f44b2cba8a2cd6d4a5faa4c665329dc Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 04:51:16 +0200 Subject: [PATCH 10/29] DiscIO/DirectoryBlob: Add a callback that allows patching the main.dol and FST of the game partition during blob construction. --- Source/Core/DiscIO/DirectoryBlob.cpp | 45 +++++++++++++++++----------- Source/Core/DiscIO/DirectoryBlob.h | 14 +++++++-- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Source/Core/DiscIO/DirectoryBlob.cpp b/Source/Core/DiscIO/DirectoryBlob.cpp index b41a00cf66..80bd173ae0 100644 --- a/Source/Core/DiscIO/DirectoryBlob.cpp +++ b/Source/Core/DiscIO/DirectoryBlob.cpp @@ -356,13 +356,16 @@ std::unique_ptr DirectoryBlobReader::Create(const std::stri return std::unique_ptr(new DirectoryBlobReader(partition_root, true_root)); } -std::unique_ptr -DirectoryBlobReader::Create(std::unique_ptr volume) +std::unique_ptr DirectoryBlobReader::Create( + std::unique_ptr volume, + const std::function* fst_nodes, FSTBuilderNode* dol_node)>& + fst_callback) { if (!volume) return nullptr; - return std::unique_ptr(new DirectoryBlobReader(std::move(volume))); + return std::unique_ptr( + new DirectoryBlobReader(std::move(volume), fst_callback)); } DirectoryBlobReader::DirectoryBlobReader(const std::string& game_partition_root, @@ -409,11 +412,14 @@ DirectoryBlobReader::DirectoryBlobReader(const std::string& game_partition_root, } } -DirectoryBlobReader::DirectoryBlobReader(std::unique_ptr volume) +DirectoryBlobReader::DirectoryBlobReader( + std::unique_ptr volume, + const std::function* fst_nodes, FSTBuilderNode* dol_node)>& + fst_callback) : m_encryption_cache(this), m_wrapped_volume(std::move(volume)) { - DirectoryBlobPartition game_partition(m_wrapped_volume.get(), - m_wrapped_volume->GetGamePartition(), std::nullopt); + DirectoryBlobPartition game_partition( + m_wrapped_volume.get(), m_wrapped_volume->GetGamePartition(), std::nullopt, fst_callback); m_is_wii = game_partition.IsWii(); if (!m_is_wii) @@ -452,8 +458,9 @@ DirectoryBlobReader::DirectoryBlobReader(std::unique_ptr vol auto type = m_wrapped_volume->GetPartitionType(partition); if (type) { - partitions.emplace_back(DirectoryBlobPartition(m_wrapped_volume.get(), partition, m_is_wii), - static_cast(*type)); + partitions.emplace_back( + DirectoryBlobPartition(m_wrapped_volume.get(), partition, m_is_wii, nullptr), + static_cast(*type)); } } @@ -809,9 +816,10 @@ DirectoryBlobPartition::DirectoryBlobPartition(const std::string& root_directory BuildFSTFromFolder(m_root_directory + "files/", fst_address); } -DirectoryBlobPartition::DirectoryBlobPartition(DiscIO::VolumeDisc* volume, - const DiscIO::Partition& partition, - std::optional is_wii) +DirectoryBlobPartition::DirectoryBlobPartition( + DiscIO::VolumeDisc* volume, const DiscIO::Partition& partition, std::optional is_wii, + const std::function* fst_nodes, FSTBuilderNode* dol_node)>& + fst_callback) : m_wrapped_partition(partition) { std::vector disc_header(DISCHEADER_SIZE); @@ -849,14 +857,17 @@ DirectoryBlobPartition::DirectoryBlobPartition(DiscIO::VolumeDisc* volume, dol_node.m_content = std::move(dol_contents); } } - const u64 new_fst_address = SetDOL(std::move(dol_node), new_dol_address); - - const FileSystem* fs = volume->GetFileSystem(partition); - if (!fs || !fs->IsValid()) - return; std::vector nodes; - GenerateBuilderNodesFromFileSystem(*volume, partition, &nodes, fs->GetRoot()); + + const FileSystem* fs = volume->GetFileSystem(partition); + if (fs && fs->IsValid()) + GenerateBuilderNodesFromFileSystem(*volume, partition, &nodes, fs->GetRoot()); + + if (fst_callback) + fst_callback(&nodes, &dol_node); + + const u64 new_fst_address = SetDOL(std::move(dol_node), new_dol_address); BuildFST(std::move(nodes), new_fst_address); } diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index db88367d8c..b8ec90b4f7 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -182,7 +183,9 @@ public: DirectoryBlobPartition() = default; DirectoryBlobPartition(const std::string& root_directory, std::optional is_wii); DirectoryBlobPartition(DiscIO::VolumeDisc* volume, const DiscIO::Partition& partition, - std::optional is_wii); + std::optional is_wii, + const std::function* fst_nodes, + FSTBuilderNode* dol_node)>& fst_callback); // We do not allow copying, because it might mess up the pointers inside DiscContents DirectoryBlobPartition(const DirectoryBlobPartition&) = delete; @@ -252,7 +255,10 @@ class DirectoryBlobReader : public BlobReader public: static std::unique_ptr Create(const std::string& dol_path); - static std::unique_ptr Create(std::unique_ptr volume); + static std::unique_ptr Create( + std::unique_ptr volume, + const std::function* fst_nodes, FSTBuilderNode* dol_node)>& + fst_callback); // We do not allow copying, because it might mess up the pointers inside DiscContents DirectoryBlobReader(const DirectoryBlobReader&) = delete; @@ -288,7 +294,9 @@ private: explicit DirectoryBlobReader(const std::string& game_partition_root, const std::string& true_root); - explicit DirectoryBlobReader(std::unique_ptr volume); + explicit DirectoryBlobReader(std::unique_ptr volume, + const std::function* fst_nodes, + FSTBuilderNode* dol_node)>& fst_callback); const DirectoryBlobPartition* GetPartition(u64 offset, u64 size, u64 partition_data_offset) const; From 09fc39e2e550c0a6cc303c9ab2aecdb021665b4c Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Wed, 22 Sep 2021 05:43:05 +0200 Subject: [PATCH 11/29] DiscIO: Add CreateDisc/WAD/Volume() overloads that take a BlobReader directly. --- Source/Core/DiscIO/Volume.cpp | 42 +++++++++++++++++++++++------------ Source/Core/DiscIO/Volume.h | 3 +++ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/Source/Core/DiscIO/Volume.cpp b/Source/Core/DiscIO/Volume.cpp index c1b1400876..bf7ef4e019 100644 --- a/Source/Core/DiscIO/Volume.cpp +++ b/Source/Core/DiscIO/Volume.cpp @@ -85,8 +85,11 @@ std::map Volume::ReadWiiNames(const std::vector return names; } -static std::unique_ptr CreateDisc(std::unique_ptr& reader) +static std::unique_ptr TryCreateDisc(std::unique_ptr& reader) { + if (!reader) + return nullptr; + if (reader->ReadSwapped(0x18) == WII_DISC_MAGIC) return std::make_unique(std::move(reader)); @@ -97,14 +100,21 @@ static std::unique_ptr CreateDisc(std::unique_ptr& reade return nullptr; } -std::unique_ptr CreateDisc(const std::string& path) +std::unique_ptr CreateDisc(std::unique_ptr reader) { - std::unique_ptr reader(CreateBlobReader(path)); - return reader ? CreateDisc(reader) : nullptr; + return TryCreateDisc(reader); } -static std::unique_ptr CreateWAD(std::unique_ptr& reader) +std::unique_ptr CreateDisc(const std::string& path) { + return CreateDisc(CreateBlobReader(path)); +} + +static std::unique_ptr TryCreateWAD(std::unique_ptr& reader) +{ + if (!reader) + return nullptr; + // Check for WAD // 0x206962 for boot2 wads const std::optional wad_magic = reader->ReadSwapped(0x02); @@ -115,27 +125,31 @@ static std::unique_ptr CreateWAD(std::unique_ptr& reader) return nullptr; } -std::unique_ptr CreateWAD(const std::string& path) +std::unique_ptr CreateWAD(std::unique_ptr reader) { - std::unique_ptr reader(CreateBlobReader(path)); - return reader ? CreateWAD(reader) : nullptr; + return TryCreateWAD(reader); } -std::unique_ptr CreateVolume(const std::string& path) +std::unique_ptr CreateWAD(const std::string& path) { - std::unique_ptr reader(CreateBlobReader(path)); - if (reader == nullptr) - return nullptr; + return CreateWAD(CreateBlobReader(path)); +} - std::unique_ptr disc = CreateDisc(reader); +std::unique_ptr CreateVolume(std::unique_ptr reader) +{ + std::unique_ptr disc = TryCreateDisc(reader); if (disc) return disc; - std::unique_ptr wad = CreateWAD(reader); + std::unique_ptr wad = TryCreateWAD(reader); if (wad) return wad; return nullptr; } +std::unique_ptr CreateVolume(const std::string& path) +{ + return CreateVolume(CreateBlobReader(path)); +} } // namespace DiscIO diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index 9e378bf879..574afd6f83 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -182,8 +182,11 @@ protected: static const std::vector INVALID_CERT_CHAIN; }; +std::unique_ptr CreateDisc(std::unique_ptr reader); std::unique_ptr CreateDisc(const std::string& path); +std::unique_ptr CreateWAD(std::unique_ptr reader); std::unique_ptr CreateWAD(const std::string& path); +std::unique_ptr CreateVolume(std::unique_ptr reader); std::unique_ptr CreateVolume(const std::string& path); } // namespace DiscIO From a4da56e5e6d5a056969e9d9ae6024a411a832ca4 Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Sun, 26 Sep 2021 06:16:55 +0200 Subject: [PATCH 12/29] CommonPaths: Add a Riivolution subfolder in Load. --- Source/Core/Common/CommonPaths.h | 1 + Source/Core/Common/FileUtil.cpp | 2 ++ Source/Core/Common/FileUtil.h | 1 + Source/Core/UICommon/UICommon.cpp | 1 + 4 files changed, 5 insertions(+) diff --git a/Source/Core/Common/CommonPaths.h b/Source/Core/Common/CommonPaths.h index 55ed606517..1482b9656c 100644 --- a/Source/Core/Common/CommonPaths.h +++ b/Source/Core/Common/CommonPaths.h @@ -47,6 +47,7 @@ #define SCREENSHOTS_DIR "ScreenShots" #define LOAD_DIR "Load" #define HIRES_TEXTURES_DIR "Textures" +#define RIIVOLUTION_DIR "Riivolution" #define DUMP_DIR "Dump" #define DUMP_TEXTURES_DIR "Textures" #define DUMP_FRAMES_DIR "Frames" diff --git a/Source/Core/Common/FileUtil.cpp b/Source/Core/Common/FileUtil.cpp index 1bc92c8d70..07d0d55c55 100644 --- a/Source/Core/Common/FileUtil.cpp +++ b/Source/Core/Common/FileUtil.cpp @@ -943,6 +943,7 @@ static void RebuildUserDirectories(unsigned int dir_index) s_user_paths[D_SCREENSHOTS_IDX] = s_user_paths[D_USER_IDX] + SCREENSHOTS_DIR DIR_SEP; s_user_paths[D_LOAD_IDX] = s_user_paths[D_USER_IDX] + LOAD_DIR DIR_SEP; s_user_paths[D_HIRESTEXTURES_IDX] = s_user_paths[D_LOAD_IDX] + HIRES_TEXTURES_DIR DIR_SEP; + s_user_paths[D_RIIVOLUTION_IDX] = s_user_paths[D_LOAD_IDX] + RIIVOLUTION_DIR DIR_SEP; s_user_paths[D_DUMP_IDX] = s_user_paths[D_USER_IDX] + DUMP_DIR DIR_SEP; s_user_paths[D_DUMPFRAMES_IDX] = s_user_paths[D_DUMP_IDX] + DUMP_FRAMES_DIR DIR_SEP; s_user_paths[D_DUMPOBJECTS_IDX] = s_user_paths[D_DUMP_IDX] + DUMP_OBJECTS_DIR DIR_SEP; @@ -1035,6 +1036,7 @@ static void RebuildUserDirectories(unsigned int dir_index) case D_LOAD_IDX: s_user_paths[D_HIRESTEXTURES_IDX] = s_user_paths[D_LOAD_IDX] + HIRES_TEXTURES_DIR DIR_SEP; + s_user_paths[D_RIIVOLUTION_IDX] = s_user_paths[D_LOAD_IDX] + RIIVOLUTION_DIR DIR_SEP; s_user_paths[D_DYNAMICINPUT_IDX] = s_user_paths[D_LOAD_IDX] + DYNAMICINPUT_DIR DIR_SEP; break; } diff --git a/Source/Core/Common/FileUtil.h b/Source/Core/Common/FileUtil.h index 46bcdb150f..415c3fd102 100644 --- a/Source/Core/Common/FileUtil.h +++ b/Source/Core/Common/FileUtil.h @@ -41,6 +41,7 @@ enum D_STATESAVES_IDX, D_SCREENSHOTS_IDX, D_HIRESTEXTURES_IDX, + D_RIIVOLUTION_IDX, D_DUMP_IDX, D_DUMPFRAMES_IDX, D_DUMPOBJECTS_IDX, diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp index cb2418c0c0..5acf58351f 100644 --- a/Source/Core/UICommon/UICommon.cpp +++ b/Source/Core/UICommon/UICommon.cpp @@ -67,6 +67,7 @@ static void CreateLoadPath(const std::string& path) if (!path.empty()) File::SetUserPath(D_LOAD_IDX, path + '/'); File::CreateFullPath(File::GetUserPath(D_HIRESTEXTURES_IDX)); + File::CreateFullPath(File::GetUserPath(D_RIIVOLUTION_IDX)); } static void CreateResourcePackPath(const std::string& path) From e26b59bab39ee5117b8a01f63d739f9608009e1c Mon Sep 17 00:00:00 2001 From: "Admiral H. Curtiss" Date: Thu, 23 Sep 2021 03:21:53 +0200 Subject: [PATCH 13/29] Core: Add RiivolutionParser to parse a Riivolution XML. --- Source/Core/DiscIO/CMakeLists.txt | 2 + Source/Core/DiscIO/RiivolutionParser.cpp | 336 +++++++++++++++++++++++ Source/Core/DiscIO/RiivolutionParser.h | 193 +++++++++++++ Source/Core/DolphinLib.props | 2 + 4 files changed, 533 insertions(+) create mode 100644 Source/Core/DiscIO/RiivolutionParser.cpp create mode 100644 Source/Core/DiscIO/RiivolutionParser.h diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 1729f0d959..9f0e7b6cde 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -28,6 +28,8 @@ add_library(discio MultithreadedCompressor.h NANDImporter.cpp NANDImporter.h + RiivolutionParser.cpp + RiivolutionParser.h ScrubbedBlob.cpp ScrubbedBlob.h TGCBlob.cpp diff --git a/Source/Core/DiscIO/RiivolutionParser.cpp b/Source/Core/DiscIO/RiivolutionParser.cpp new file mode 100644 index 0000000000..3162c40d43 --- /dev/null +++ b/Source/Core/DiscIO/RiivolutionParser.cpp @@ -0,0 +1,336 @@ +// Copyright 2021 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "DiscIO/RiivolutionParser.h" + +#include +#include +#include +#include + +#include + +#include "Common/FileUtil.h" +#include "Common/IOFile.h" +#include "Common/StringUtil.h" + +namespace DiscIO::Riivolution +{ +std::optional ParseFile(const std::string& filename) +{ + ::File::IOFile f(filename, "rb"); + if (!f) + return std::nullopt; + + std::vector data; + data.resize(f.GetSize()); + if (!f.ReadBytes(data.data(), data.size())) + return std::nullopt; + + return ParseString(std::string_view(data.data(), data.size()), filename); +} + +static std::map ReadParams(const pugi::xml_node& node, + std::map params = {}) +{ + for (const auto& param_node : node.children("param")) + { + const std::string param_name = param_node.attribute("name").as_string(); + const std::string param_value = param_node.attribute("value").as_string(); + params[param_name] = param_value; + } + return params; +} + +static std::vector ReadHexString(std::string_view sv) +{ + if ((sv.size() % 2) == 1) + return {}; + if (StringBeginsWith(sv, "0x") || StringBeginsWith(sv, "0X")) + sv = sv.substr(2); + + std::vector result; + result.reserve(sv.size() / 2); + while (!sv.empty()) + { + u8 tmp; + if (!TryParse(std::string(sv.substr(0, 2)), &tmp, 16)) + return {}; + result.push_back(tmp); + sv = sv.substr(2); + } + return result; +}; + +std::optional ParseString(std::string_view xml, std::string xml_path) +{ + pugi::xml_document doc; + const auto parse_result = doc.load_buffer(xml.data(), xml.size()); + if (!parse_result) + return std::nullopt; + + const auto wiidisc = doc.child("wiidisc"); + if (!wiidisc) + return std::nullopt; + + Disc disc; + disc.m_xml_path = std::move(xml_path); + disc.m_version = wiidisc.attribute("version").as_int(-1); + if (disc.m_version != 1) + return std::nullopt; + const std::string default_root = wiidisc.attribute("root").as_string(); + + const auto id = wiidisc.child("id"); + if (id) + { + for (const auto& attribute : id.attributes()) + { + const std::string_view attribute_name(attribute.name()); + if (attribute_name == "game") + disc.m_game_filter.m_game = attribute.as_string(); + else if (attribute_name == "developer") + disc.m_game_filter.m_developer = attribute.as_string(); + else if (attribute_name == "disc") + disc.m_game_filter.m_disc = attribute.as_int(-1); + else if (attribute_name == "version") + disc.m_game_filter.m_version = attribute.as_int(-1); + } + + auto xml_regions = id.children("region"); + if (xml_regions.begin() != xml_regions.end()) + { + std::vector regions; + for (const auto& region : xml_regions) + regions.push_back(region.attribute("type").as_string()); + disc.m_game_filter.m_regions = std::move(regions); + } + } + + const auto options = wiidisc.child("options"); + if (options) + { + for (const auto& section_node : options.children("section")) + { + Section& section = disc.m_sections.emplace_back(); + section.m_name = section_node.attribute("name").as_string(); + for (const auto& option_node : section_node.children("option")) + { + Option& option = section.m_options.emplace_back(); + option.m_id = option_node.attribute("id").as_string(); + option.m_name = option_node.attribute("name").as_string(); + option.m_selected_choice = option_node.attribute("default").as_uint(0); + auto option_params = ReadParams(option_node); + for (const auto& choice_node : option_node.children("choice")) + { + Choice& choice = option.m_choices.emplace_back(); + choice.m_name = choice_node.attribute("name").as_string(); + auto choice_params = ReadParams(choice_node, option_params); + for (const auto& patchref_node : choice_node.children("patch")) + { + PatchReference& patchref = choice.m_patch_references.emplace_back(); + patchref.m_id = patchref_node.attribute("id").as_string(); + patchref.m_params = ReadParams(patchref_node, choice_params); + } + } + } + } + for (const auto& macro_node : options.children("macros")) + { + const std::string macro_id = macro_node.attribute("id").as_string(); + for (auto& section : disc.m_sections) + { + auto option_to_clone = std::find_if(section.m_options.begin(), section.m_options.end(), + [&](const Option& o) { return o.m_id == macro_id; }); + if (option_to_clone == section.m_options.end()) + continue; + + Option cloned_option = *option_to_clone; + cloned_option.m_name = macro_node.attribute("name").as_string(); + for (auto& choice : cloned_option.m_choices) + for (auto& patch_ref : choice.m_patch_references) + patch_ref.m_params = ReadParams(macro_node, patch_ref.m_params); + } + } + } + + const auto patches = wiidisc.children("patch"); + for (const auto& patch_node : patches) + { + Patch& patch = disc.m_patches.emplace_back(); + patch.m_id = patch_node.attribute("id").as_string(); + patch.m_root = patch_node.attribute("root").as_string(); + if (patch.m_root.empty()) + patch.m_root = default_root; + + for (const auto& patch_subnode : patch_node.children()) + { + const std::string_view patch_name(patch_subnode.name()); + if (patch_name == "file") + { + auto& file = patch.m_file_patches.emplace_back(); + file.m_disc = patch_subnode.attribute("disc").as_string(); + file.m_external = patch_subnode.attribute("external").as_string(); + file.m_resize = patch_subnode.attribute("resize").as_bool(true); + file.m_create = patch_subnode.attribute("create").as_bool(false); + file.m_offset = patch_subnode.attribute("offset").as_uint(0); + file.m_fileoffset = patch_subnode.attribute("fileoffset").as_uint(0); + file.m_length = patch_subnode.attribute("length").as_uint(0); + } + else if (patch_name == "folder") + { + auto& folder = patch.m_folder_patches.emplace_back(); + folder.m_disc = patch_subnode.attribute("disc").as_string(); + folder.m_external = patch_subnode.attribute("external").as_string(); + folder.m_resize = patch_subnode.attribute("resize").as_bool(true); + folder.m_create = patch_subnode.attribute("create").as_bool(false); + folder.m_recursive = patch_subnode.attribute("recursive").as_bool(true); + folder.m_length = patch_subnode.attribute("length").as_uint(0); + } + else if (patch_name == "savegame") + { + auto& savegame = patch.m_savegame_patches.emplace_back(); + savegame.m_external = patch_subnode.attribute("external").as_string(); + savegame.m_clone = patch_subnode.attribute("clone").as_bool(true); + } + else if (patch_name == "memory") + { + auto& memory = patch.m_memory_patches.emplace_back(); + memory.m_offset = patch_subnode.attribute("offset").as_uint(0); + memory.m_value = ReadHexString(patch_subnode.attribute("value").as_string()); + memory.m_valuefile = patch_subnode.attribute("valuefile").as_string(); + memory.m_original = ReadHexString(patch_subnode.attribute("original").as_string()); + memory.m_ocarina = patch_subnode.attribute("ocarina").as_bool(false); + memory.m_search = patch_subnode.attribute("search").as_bool(false); + memory.m_align = patch_subnode.attribute("align").as_uint(1); + } + } + } + + return disc; +} + +static bool CheckRegion(const std::vector& xml_regions, std::string_view game_region) +{ + if (xml_regions.begin() == xml_regions.end()) + return true; + + for (const auto& region : xml_regions) + { + if (region == game_region) + return true; + } + + return false; +} + +bool Disc::IsValidForGame(const std::string& game_id, std::optional revision, + std::optional disc_number) const +{ + if (game_id.size() != 6) + return false; + + const std::string_view game_id_full = std::string_view(game_id); + const std::string_view game_region = game_id_full.substr(3, 1); + const std::string_view game_developer = game_id_full.substr(4, 2); + const int disc_number_int = std::optional(disc_number).value_or(-1); + const int revision_int = std::optional(revision).value_or(-1); + + if (m_game_filter.m_game && !StringBeginsWith(game_id_full, *m_game_filter.m_game)) + return false; + if (m_game_filter.m_developer && game_developer != *m_game_filter.m_developer) + return false; + if (m_game_filter.m_disc && disc_number_int != *m_game_filter.m_disc) + return false; + if (m_game_filter.m_version && revision_int != *m_game_filter.m_version) + return false; + if (m_game_filter.m_regions && !CheckRegion(*m_game_filter.m_regions, game_region)) + return false; + + return true; +} + +std::vector Disc::GeneratePatches(const std::string& game_id) const +{ + const std::string_view game_id_full = std::string_view(game_id); + const std::string_view game_id_no_region = game_id_full.substr(0, 3); + const std::string_view game_region = game_id_full.substr(3, 1); + const std::string_view game_developer = game_id_full.substr(4, 2); + + const auto replace_variables = + [&](std::string_view sv, + const std::vector>& replacements) { + std::string result; + result.reserve(sv.size()); + while (!sv.empty()) + { + bool replaced = false; + for (const auto& r : replacements) + { + if (StringBeginsWith(sv, r.first)) + { + for (char c : r.second) + result.push_back(c); + sv = sv.substr(r.first.size()); + replaced = true; + break; + } + } + if (replaced) + continue; + result.push_back(sv[0]); + sv = sv.substr(1); + } + return result; + }; + + // Take only selected patches, replace placeholders in all strings, and return them. + std::vector active_patches; + for (const auto& section : m_sections) + { + for (const auto& option : section.m_options) + { + const u32 selected = option.m_selected_choice; + if (selected == 0 || selected > option.m_choices.size()) + continue; + const Choice& choice = option.m_choices[selected - 1]; + for (const auto& patch_ref : choice.m_patch_references) + { + const auto patch = std::find_if(m_patches.begin(), m_patches.end(), + [&](const Patch& p) { return patch_ref.m_id == p.m_id; }); + if (patch == m_patches.end()) + continue; + + std::vector> replacements; + replacements.emplace_back(std::pair{"{$__gameid}", game_id_no_region}); + replacements.emplace_back(std::pair{"{$__region}", game_region}); + replacements.emplace_back(std::pair{"{$__maker}", game_developer}); + for (const auto& param : patch_ref.m_params) + replacements.emplace_back(std::pair{"{$" + param.first + "}", param.second}); + + Patch& new_patch = active_patches.emplace_back(*patch); + new_patch.m_root = replace_variables(new_patch.m_root, replacements); + for (auto& file : new_patch.m_file_patches) + { + file.m_disc = replace_variables(file.m_disc, replacements); + file.m_external = replace_variables(file.m_external, replacements); + } + for (auto& folder : new_patch.m_folder_patches) + { + folder.m_disc = replace_variables(folder.m_disc, replacements); + folder.m_external = replace_variables(folder.m_external, replacements); + } + for (auto& savegame : new_patch.m_savegame_patches) + { + savegame.m_external = replace_variables(savegame.m_external, replacements); + } + for (auto& memory : new_patch.m_memory_patches) + { + memory.m_valuefile = replace_variables(memory.m_valuefile, replacements); + } + } + } + } + + return active_patches; +} +} // namespace DiscIO::Riivolution diff --git a/Source/Core/DiscIO/RiivolutionParser.h b/Source/Core/DiscIO/RiivolutionParser.h new file mode 100644 index 0000000000..61b81be569 --- /dev/null +++ b/Source/Core/DiscIO/RiivolutionParser.h @@ -0,0 +1,193 @@ +// Copyright 2021 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include +#include + +#include "Common/CommonTypes.h" + +namespace DiscIO::Riivolution +{ +// Data to determine the game patches are valid for. +struct GameFilter +{ + std::optional m_game; + std::optional m_developer; + std::optional m_disc; + std::optional m_version; + std::optional> m_regions; +}; + +// Which patches will get activated by selecting a Choice in the Riivolution GUI. +struct PatchReference +{ + std::string m_id; + std::map m_params; +}; + +// A single choice within an Option in the Riivolution GUI. +struct Choice +{ + std::string m_name; + std::vector m_patch_references; +}; + +// A single option presented to the user in the Riivolution GUI. +struct Option +{ + std::string m_name; + std::string m_id; + std::vector m_choices; + + // The currently selected patch choice in the m_choices vector. + // Note that this index is 1-based; 0 means no choice is selected and this Option is disabled. + u32 m_selected_choice; +}; + +// A single page of options presented to the user in the Riivolution GUI. +struct Section +{ + std::string m_name; + std::vector