From 4fd2d8e8c4db14cf01576ab228964b80c8ec37ad Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 26 Mar 2019 17:22:18 +0100 Subject: [PATCH] VolumeVerifier: Check hashes in Wii partitions --- Source/Core/DiscIO/Volume.h | 6 +- Source/Core/DiscIO/VolumeVerifier.cpp | 95 ++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 14 ++ Source/Core/DiscIO/VolumeWii.cpp | 138 ++++++++++-------- Source/Core/DiscIO/VolumeWii.h | 8 +- .../DolphinQt/Config/FilesystemWidget.cpp | 38 ----- .../Core/DolphinQt/Config/FilesystemWidget.h | 1 - 7 files changed, 184 insertions(+), 116 deletions(-) diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index e5e77d4479..5cd91ce60e 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -99,7 +99,11 @@ public: } virtual Platform GetVolumeType() const = 0; virtual bool SupportsIntegrityCheck() const { return false; } - virtual bool CheckIntegrity(const Partition& partition) const { return false; } + virtual bool CheckH3TableIntegrity(const Partition& partition) const { return false; } + virtual bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const + { + return false; + } virtual Region GetRegion() const = 0; virtual Country GetCountry(const Partition& partition = PARTITION_NONE) const = 0; virtual BlobType GetBlobType() const = 0; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 7d3eba4c8f..2201784d1c 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -14,6 +14,7 @@ #include "Common/Align.h" #include "Common/Assert.h" #include "Common/CommonTypes.h" +#include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "Common/StringUtil.h" #include "Common/Swap.h" @@ -62,6 +63,9 @@ void VolumeVerifier::Start() CheckCorrectlySigned(PARTITION_NONE, GetStringT("This title is not correctly signed.")); CheckDiscSize(); CheckMisc(); + + std::sort(m_blocks.begin(), m_blocks.end(), + [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); } void VolumeVerifier::CheckPartitions() @@ -164,18 +168,7 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) else if (*type == PARTITION_UPDATE) severity = Severity::Low; - std::string name = NameForPartitionType(*type, false); - if (ShouldHaveMasterpiecePartitions() && *type > 0xFF) - { - // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces - // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo - // rather than the game mode as a whole, so please use the singular form. Official translations: - // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre - // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). - // If your language is not one of the languages above, consider leaving the string untranslated - // so that people will recognize it as the name of the game mode. - name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str()); - } + const std::string name = GetPartitionName(type); if (partition.offset % VolumeWii::BLOCK_TOTAL_SIZE != 0 || m_volume.PartitionOffsetToRawOffset(0, partition) % VolumeWii::BLOCK_TOTAL_SIZE != 0) @@ -189,6 +182,13 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) partition, StringFromFormat(GetStringT("The %s partition is not correctly signed.").c_str(), name.c_str())); + if (m_volume.SupportsIntegrityCheck() && !m_volume.CheckH3TableIntegrity(partition)) + { + const std::string text = StringFromFormat( + GetStringT("The H3 hash table for the %s partition is not correct.").c_str(), name.c_str()); + AddProblem(Severity::Low, text); + } + bool invalid_disc_header = false; std::vector disc_header(0x80); constexpr u32 WII_MAGIC = 0x5D1C9EA3; @@ -262,9 +262,43 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) } } + // Prepare for hash verification in the Process step + if (m_volume.SupportsIntegrityCheck()) + { + u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition); + const std::optional size = + m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE); + const u64 end_offset = offset + size.value_or(0); + + for (size_t i = 0; offset < end_offset; ++i, offset += VolumeWii::BLOCK_TOTAL_SIZE) + m_blocks.emplace_back(BlockToVerify{partition, offset, i}); + + m_block_errors.emplace(partition, 0); + } + return true; } +std::string VolumeVerifier::GetPartitionName(std::optional type) const +{ + if (!type) + return "???"; + + std::string name = NameForPartitionType(*type, false); + if (ShouldHaveMasterpiecePartitions() && *type > 0xFF) + { + // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces + // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo + // rather than the game mode as a whole, so please use the singular form. Official translations: + // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre + // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). + // If your language is not one of the languages above, consider leaving the string untranslated + // so that people will recognize it as the name of the game mode. + name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str()); + } + return name; +} + void VolumeVerifier::CheckCorrectlySigned(const Partition& partition, const std::string& error_text) { IOS::HLE::Kernel ios; @@ -596,7 +630,30 @@ void VolumeVerifier::Process() if (m_progress == m_max_progress) return; - m_progress += std::min(m_max_progress - m_progress, BLOCK_SIZE); + u64 bytes_to_read = BLOCK_SIZE; + if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress) + { + bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE; + } + else if (m_block_index + 1 < m_blocks.size() && m_blocks[m_block_index + 1].offset > m_progress) + { + bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index + 1].offset - m_progress); + } + bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress); + + m_progress += bytes_to_read; + + while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress) + { + if (!m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index, + m_blocks[m_block_index].partition)) + { + WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, + m_blocks[m_block_index].offset); + m_block_errors[m_blocks[m_block_index].partition]++; + } + m_block_index++; + } } u64 VolumeVerifier::GetBytesProcessed() const @@ -615,6 +672,18 @@ void VolumeVerifier::Finish() return; m_done = true; + for (auto pair : m_block_errors) + { + if (pair.second > 0) + { + const std::string name = GetPartitionName(m_volume.GetPartitionType(pair.first)); + AddProblem(Severity::Medium, + StringFromFormat( + GetStringT("Errors were found in %zu blocks in the %s partition.").c_str(), + pair.second, name.c_str())); + } + } + // Show the most serious problems at the top std::stable_sort(m_result.problems.begin(), m_result.problems.end(), [](const Problem& p1, const Problem& p2) { return p1.severity > p2.severity; }); diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 8503c1f89d..22b86107c0 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -4,6 +4,8 @@ #pragma once +#include +#include #include #include @@ -64,8 +66,16 @@ public: const Result& GetResult() const; private: + struct BlockToVerify + { + Partition partition; + u64 offset; + u64 block_index; + }; + void CheckPartitions(); bool CheckPartition(const Partition& partition); // Returns false if partition should be ignored + std::string GetPartitionName(std::optional type) const; void CheckCorrectlySigned(const Partition& partition, const std::string& error_text); bool IsDebugSigned() const; bool ShouldHaveChannelPartition() const; @@ -85,6 +95,10 @@ private: bool m_is_datel; bool m_is_not_retail; + std::vector m_blocks; + size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition + std::map m_block_errors; + bool m_started; bool m_done; u64 m_progress; diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index f589c69921..7d44315bf1 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -109,6 +109,19 @@ VolumeWii::VolumeWii(std::unique_ptr reader) return cert_chain; }; + auto get_h3_table = [this, partition]() -> std::vector { + if (!m_encrypted) + return {}; + const std::optional h3_table_offset = + ReadSwappedAndShifted(partition.offset + 0x2b4, PARTITION_NONE); + if (!h3_table_offset) + return {}; + std::vector h3_table(H3_TABLE_SIZE); + if (!m_reader->Read(partition.offset + *h3_table_offset, H3_TABLE_SIZE, h3_table.data())) + return {}; + return h3_table; + }; + auto get_key = [this, partition]() -> std::unique_ptr { const IOS::ES::TicketReader& ticket = *m_partitions[partition].ticket; if (!ticket.IsValid()) @@ -133,6 +146,7 @@ VolumeWii::VolumeWii(std::unique_ptr reader) Common::Lazy(get_ticket), Common::Lazy(get_tmd), Common::Lazy>(get_cert_chain), + Common::Lazy>(get_h3_table), Common::Lazy>(get_file_system), Common::Lazy(get_data_offset), *partition_type}); } @@ -409,82 +423,84 @@ u64 VolumeWii::GetRawSize() const return m_reader->GetRawSize(); } -bool VolumeWii::CheckIntegrity(const Partition& partition) const +bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const { - if (!m_encrypted) - return false; - - // Get the decryption key for the partition auto it = m_partitions.find(partition); if (it == m_partitions.end()) return false; const PartitionDetails& partition_details = it->second; + + const std::vector& h3_table = *partition_details.h3_table; + if (h3_table.size() != H3_TABLE_SIZE) + return false; + + const IOS::ES::TMDReader& tmd = *partition_details.tmd; + if (!tmd.IsValid()) + return false; + + const std::vector contents = tmd.GetContents(); + if (contents.size() != 1) + return false; + + std::array h3_table_sha1; + mbedtls_sha1(h3_table.data(), h3_table.size(), h3_table_sha1.data()); + return h3_table_sha1 == contents[0].sha1; +} + +bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition) const +{ + auto it = m_partitions.find(partition); + if (it == m_partitions.end()) + return false; + const PartitionDetails& partition_details = it->second; + + constexpr size_t SHA1_SIZE = 20; + if (block_index / 64 * SHA1_SIZE >= partition_details.h3_table->size()) + return false; + mbedtls_aes_context* aes_context = partition_details.key->get(); if (!aes_context) return false; - // Get partition data size - const auto part_data_size = ReadSwappedAndShifted(partition.offset + 0x2BC, PARTITION_NONE); - if (!part_data_size) + const u64 cluster_offset = + partition.offset + *partition_details.data_offset + block_index * BLOCK_TOTAL_SIZE; + + // Read and decrypt the cluster metadata + u8 cluster_metadata_crypted[BLOCK_HEADER_SIZE]; + u8 cluster_metadata[BLOCK_HEADER_SIZE]; + u8 iv[16] = {0}; + if (!m_reader->Read(cluster_offset, BLOCK_HEADER_SIZE, cluster_metadata_crypted)) + return false; + mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_HEADER_SIZE, iv, + cluster_metadata_crypted, cluster_metadata); + + u8 cluster_data[BLOCK_DATA_SIZE]; + if (!Read(block_index * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE, cluster_data, partition)) return false; - const u32 num_clusters = static_cast(part_data_size.value() / 0x8000); - for (u32 cluster_id = 0; cluster_id < num_clusters; ++cluster_id) + for (u32 hash_index = 0; hash_index < 31; ++hash_index) { - const u64 cluster_offset = - partition.offset + *partition_details.data_offset + static_cast(cluster_id) * 0x8000; - - // Read and decrypt the cluster metadata - u8 cluster_metadata_crypted[0x400]; - u8 cluster_metadata[0x400]; - u8 iv[16] = {0}; - if (!m_reader->Read(cluster_offset, sizeof(cluster_metadata_crypted), cluster_metadata_crypted)) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read metadata", cluster_id); + u8 h0_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_data + hash_index * 0x400, 0x400, h0_hash); + if (memcmp(h0_hash, cluster_metadata + hash_index * SHA1_SIZE, SHA1_SIZE)) return false; - } - mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_metadata), iv, - cluster_metadata_crypted, cluster_metadata); - - // Some clusters have invalid data and metadata because they aren't - // meant to be read by the game (for example, holes between files). To - // try to avoid reporting errors because of these clusters, we check - // the 0x00 paddings in the metadata. - // - // This may cause some false negatives though: some bad clusters may be - // skipped because they are *too* bad and are not even recognized as - // valid clusters. To be improved. - const u8* pad_begin = cluster_metadata + 0x26C; - const u8* pad_end = pad_begin + 0x14; - const bool meaningless = std::any_of(pad_begin, pad_end, [](u8 val) { return val != 0; }); - - if (meaningless) - continue; - - u8 cluster_data[0x7C00]; - if (!Read(cluster_id * sizeof(cluster_data), sizeof(cluster_data), cluster_data, partition)) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read data", cluster_id); - return false; - } - - for (u32 hash_id = 0; hash_id < 31; ++hash_id) - { - u8 hash[20]; - - mbedtls_sha1(cluster_data + hash_id * sizeof(cluster_metadata), sizeof(cluster_metadata), - hash); - - // Note that we do not use strncmp here - if (memcmp(hash, cluster_metadata + hash_id * sizeof(hash), sizeof(hash))) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: hash %d is invalid", cluster_id, - hash_id); - return false; - } - } } + u8 h1_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata, SHA1_SIZE * 31, h1_hash); + if (memcmp(h1_hash, cluster_metadata + 0x280 + (block_index % 8) * SHA1_SIZE, SHA1_SIZE)) + return false; + + u8 h2_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata + 0x280, SHA1_SIZE * 8, h2_hash); + if (memcmp(h2_hash, cluster_metadata + 0x340 + (block_index / 8 % 8) * SHA1_SIZE, SHA1_SIZE)) + return false; + + u8 h3_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata + 0x340, SHA1_SIZE * 8, h3_hash); + if (memcmp(h3_hash, partition_details.h3_table->data() + block_index / 64 * SHA1_SIZE, SHA1_SIZE)) + return false; + return true; } diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index 01df4f053f..edfa75d21e 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -56,8 +56,9 @@ public: std::optional GetDiscNumber(const Partition& partition = PARTITION_NONE) const override; Platform GetVolumeType() const override; - bool SupportsIntegrityCheck() const override { return true; } - bool CheckIntegrity(const Partition& partition) const override; + bool SupportsIntegrityCheck() const override { return m_encrypted; } + bool CheckH3TableIntegrity(const Partition& partition) const override; + bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const override; Region GetRegion() const override; Country GetCountry(const Partition& partition = PARTITION_NONE) const override; @@ -66,6 +67,8 @@ public: bool IsSizeAccurate() const override; u64 GetRawSize() const override; + static constexpr unsigned int H3_TABLE_SIZE = 0x18000; + static constexpr unsigned int BLOCK_HEADER_SIZE = 0x0400; static constexpr unsigned int BLOCK_DATA_SIZE = 0x7C00; static constexpr unsigned int BLOCK_TOTAL_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE; @@ -80,6 +83,7 @@ private: Common::Lazy ticket; Common::Lazy tmd; Common::Lazy> cert_chain; + Common::Lazy> h3_table; Common::Lazy> file_system; Common::Lazy data_offset; u32 type; diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp index e407ed9945..65caa17bf9 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp @@ -238,12 +238,6 @@ void FilesystemWidget::ShowContextMenu(const QPoint&) if (!folder.isEmpty()) ExtractPartition(partition, folder); }); - if (m_volume->IsEncryptedAndHashed()) - { - menu->addSeparator(); - menu->addAction(tr("Check Partition Integrity"), this, - [this, partition] { CheckIntegrity(partition); }); - } break; case EntryType::File: menu->addAction(tr("Extract File..."), this, [this, partition, path] { @@ -327,35 +321,3 @@ void FilesystemWidget::ExtractFile(const DiscIO::Partition& partition, const QSt else ModalMessageBox::critical(this, tr("Error"), tr("Failed to extract file.")); } - -void FilesystemWidget::CheckIntegrity(const DiscIO::Partition& partition) -{ - QProgressDialog* dialog = new QProgressDialog(this); - std::future is_valid = std::async( - std::launch::async, [this, partition] { return m_volume->CheckIntegrity(partition); }); - - dialog->setLabelText(tr("Verifying integrity of partition...")); - dialog->setWindowFlags(dialog->windowFlags() & ~Qt::WindowContextHelpButtonHint); - dialog->setWindowTitle(tr("Verifying partition")); - - dialog->setMinimum(0); - dialog->setMaximum(0); - dialog->show(); - - while (is_valid.wait_for(std::chrono::milliseconds(50)) != std::future_status::ready) - QCoreApplication::processEvents(); - - dialog->close(); - - if (is_valid.get()) - { - ModalMessageBox::information(this, tr("Success"), - tr("Integrity check completed. No errors have been found.")); - } - else - { - ModalMessageBox::critical(this, tr("Error"), - tr("Integrity check for partition failed. The disc image is most " - "likely corrupted or has been patched incorrectly.")); - } -} diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.h b/Source/Core/DolphinQt/Config/FilesystemWidget.h index c5b14adad9..99629444cf 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.h +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.h @@ -43,7 +43,6 @@ private: const QString& out); void ExtractFile(const DiscIO::Partition& partition, const QString& path, const QString& out); bool ExtractSystemData(const DiscIO::Partition& partition, const QString& out); - void CheckIntegrity(const DiscIO::Partition& partition); DiscIO::Partition GetPartitionFromID(int id);