From 2652aed85c0ab0ee9d51cfd5065b4bb4a620ab45 Mon Sep 17 00:00:00 2001 From: Pokechu22 Date: Fri, 10 Dec 2021 12:47:03 -0800 Subject: [PATCH 1/4] Common: Merge CRC32.h into Hash.h This makes it easier to find the relevant functions. --- Source/Core/Common/CMakeLists.txt | 2 -- Source/Core/Common/CRC32.cpp | 19 ------------------- Source/Core/Common/CRC32.h | 11 ----------- Source/Core/Common/Hash.cpp | 12 ++++++++++++ Source/Core/Common/Hash.h | 3 +++ Source/Core/Core/Boot/Boot.cpp | 2 +- Source/Core/DolphinLib.props | 2 -- 7 files changed, 16 insertions(+), 35 deletions(-) delete mode 100644 Source/Core/Common/CRC32.cpp delete mode 100644 Source/Core/Common/CRC32.h diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index 669a629676..a2156c5b5d 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -25,8 +25,6 @@ add_library(common Config/Layer.cpp Config/Layer.h CPUDetect.h - CRC32.cpp - CRC32.h Crypto/AES.cpp Crypto/AES.h Crypto/bn.cpp diff --git a/Source/Core/Common/CRC32.cpp b/Source/Core/Common/CRC32.cpp deleted file mode 100644 index c09a578196..0000000000 --- a/Source/Core/Common/CRC32.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2021 Dolphin Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#include "Common/CRC32.h" - -#include - -namespace Common -{ -u32 ComputeCRC32(std::string_view data) -{ - const Bytef* buf = reinterpret_cast(data.data()); - uInt len = static_cast(data.size()); - // Use zlibs crc32 implementation to compute the hash - u32 hash = crc32(0L, Z_NULL, 0); - hash = crc32(hash, buf, len); - return hash; -} -} // namespace Common diff --git a/Source/Core/Common/CRC32.h b/Source/Core/Common/CRC32.h deleted file mode 100644 index 3f3f52b96a..0000000000 --- a/Source/Core/Common/CRC32.h +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2021 Dolphin Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#include - -#include "Common/CommonTypes.h" - -namespace Common -{ -u32 ComputeCRC32(std::string_view data); -} // namespace Common diff --git a/Source/Core/Common/Hash.cpp b/Source/Core/Common/Hash.cpp index d0b05ec7b6..692745cb0a 100644 --- a/Source/Core/Common/Hash.cpp +++ b/Source/Core/Common/Hash.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include "Common/BitUtils.h" #include "Common/CPUDetect.h" #include "Common/CommonFuncs.h" @@ -529,4 +531,14 @@ void SetHash64Function() ptrHashFunction = &GetMurmurHash3; } } + +u32 ComputeCRC32(std::string_view data) +{ + const Bytef* buf = reinterpret_cast(data.data()); + uInt len = static_cast(data.size()); + // Use zlib's crc32 implementation to compute the hash + u32 hash = crc32(0L, Z_NULL, 0); + hash = crc32(hash, buf, len); + return hash; +} } // namespace Common diff --git a/Source/Core/Common/Hash.h b/Source/Core/Common/Hash.h index 6e07a82f23..c346c343e4 100644 --- a/Source/Core/Common/Hash.h +++ b/Source/Core/Common/Hash.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include "Common/CommonTypes.h" @@ -14,4 +15,6 @@ u32 HashAdler32(const u8* data, size_t len); // Fairly accurate, slightl u32 HashEctor(const u8* ptr, size_t length); // JUNK. DO NOT USE FOR NEW THINGS u64 GetHash64(const u8* src, u32 len, u32 samples); void SetHash64Function(); + +u32 ComputeCRC32(std::string_view data); } // namespace Common diff --git a/Source/Core/Core/Boot/Boot.cpp b/Source/Core/Core/Boot/Boot.cpp index 22f9b4f017..a08261b874 100644 --- a/Source/Core/Core/Boot/Boot.cpp +++ b/Source/Core/Core/Boot/Boot.cpp @@ -22,11 +22,11 @@ namespace fs = std::filesystem; #include "Common/Align.h" #include "Common/CDUtils.h" -#include "Common/CRC32.h" #include "Common/CommonPaths.h" #include "Common/CommonTypes.h" #include "Common/Config/Config.h" #include "Common/FileUtil.h" +#include "Common/Hash.h" #include "Common/IOFile.h" #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 0940466a2a..6c267853df 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -21,7 +21,6 @@ - @@ -687,7 +686,6 @@ - From 0c19f895d39b3302488acf9b6006fb524fb0e4d6 Mon Sep 17 00:00:00 2001 From: Pokechu22 Date: Fri, 10 Dec 2021 13:34:40 -0800 Subject: [PATCH 2/4] Replace remaining uses of zlib crc32 with Common/Hash.h --- Source/Core/Common/Hash.cpp | 24 +++++++++++++++---- Source/Core/Common/Hash.h | 3 +++ Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp | 8 +++---- Source/Core/DiscIO/VolumeVerifier.cpp | 7 +++--- Source/Core/DiscIO/VolumeVerifier.h | 2 +- .../DualShockUDPClient/DualShockUDPProto.h | 17 +++++++------ 6 files changed, 38 insertions(+), 23 deletions(-) diff --git a/Source/Core/Common/Hash.cpp b/Source/Core/Common/Hash.cpp index 692745cb0a..62b56bc82c 100644 --- a/Source/Core/Common/Hash.cpp +++ b/Source/Core/Common/Hash.cpp @@ -534,11 +534,25 @@ void SetHash64Function() u32 ComputeCRC32(std::string_view data) { - const Bytef* buf = reinterpret_cast(data.data()); - uInt len = static_cast(data.size()); + return ComputeCRC32(reinterpret_cast(data.data()), static_cast(data.size())); +} + +u32 ComputeCRC32(const u8* ptr, u32 length) +{ + return UpdateCRC32(StartCRC32(), ptr, length); +} + +u32 StartCRC32() +{ + return crc32(0L, Z_NULL, 0); +} + +u32 UpdateCRC32(u32 crc, const u8* ptr, u32 length) +{ + static_assert(std::is_same_v); + static_assert(std::is_same_v); // Use zlib's crc32 implementation to compute the hash - u32 hash = crc32(0L, Z_NULL, 0); - hash = crc32(hash, buf, len); - return hash; + // crc32_z (which takes a size_t) would be better, but it isn't available on Android + return crc32(crc, ptr, length); } } // namespace Common diff --git a/Source/Core/Common/Hash.h b/Source/Core/Common/Hash.h index c346c343e4..c742ed0b64 100644 --- a/Source/Core/Common/Hash.h +++ b/Source/Core/Common/Hash.h @@ -17,4 +17,7 @@ u64 GetHash64(const u8* src, u32 len, u32 samples); void SetHash64Function(); u32 ComputeCRC32(std::string_view data); +u32 ComputeCRC32(const u8* ptr, u32 length); +u32 StartCRC32(); +u32 UpdateCRC32(u32 crc, const u8* ptr, u32 length); } // namespace Common diff --git a/Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp b/Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp index 72c5eab82b..7b4f0be1a3 100644 --- a/Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp +++ b/Source/Core/Core/HW/WiimoteEmu/MotionPlus.cpp @@ -8,10 +8,10 @@ #include #include -#include #include "Common/BitUtils.h" #include "Common/ChunkFile.h" +#include "Common/Hash.h" #include "Common/Logging/Log.h" #include "Common/MathUtil.h" #include "Common/MsgHandler.h" @@ -146,9 +146,9 @@ void MotionPlus::Reset() void MotionPlus::CalibrationData::UpdateChecksum() { // Checksum is crc32 of all data other than the checksum itself. - auto crc_result = crc32(0, Z_NULL, 0); - crc_result = crc32(crc_result, reinterpret_cast(this), 0xe); - crc_result = crc32(crc_result, reinterpret_cast(this) + 0x10, 0xe); + u32 crc_result = Common::StartCRC32(); + crc_result = Common::UpdateCRC32(crc_result, reinterpret_cast(this), 0xe); + crc_result = Common::UpdateCRC32(crc_result, reinterpret_cast(this) + 0x10, 0xe); crc32_lsb = u16(crc_result); crc32_msb = u16(crc_result >> 16); diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index a31243c006..0a17864290 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -16,13 +16,13 @@ #include #include #include -#include #include "Common/Align.h" #include "Common/Assert.h" #include "Common/CommonPaths.h" #include "Common/CommonTypes.h" #include "Common/FileUtil.h" +#include "Common/Hash.h" #include "Common/HttpRequest.h" #include "Common/IOFile.h" #include "Common/Logging/Log.h" @@ -1041,7 +1041,7 @@ void VolumeVerifier::SetUpHashing() [](const GroupToVerify& a, const GroupToVerify& b) { return a.offset < b.offset; }); if (m_hashes_to_calculate.crc32) - m_crc32_context = crc32(0, nullptr, 0); + m_crc32_context = Common::StartCRC32(); if (m_hashes_to_calculate.md5) { @@ -1171,9 +1171,8 @@ void VolumeVerifier::Process() if (m_hashes_to_calculate.crc32) { m_crc32_future = std::async(std::launch::async, [this, byte_increment] { - // It would be nice to use crc32_z here instead of crc32, but it isn't available on Android m_crc32_context = - crc32(m_crc32_context, m_data.data(), static_cast(byte_increment)); + Common::UpdateCRC32(m_crc32_context, m_data.data(), static_cast(byte_increment)); }); } diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 6e573880ac..1a38f610b9 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -172,7 +172,7 @@ private: Hashes m_hashes_to_calculate{}; bool m_calculating_any_hash = false; - unsigned long m_crc32_context = 0; + u32 m_crc32_context = 0; mbedtls_md5_context m_md5_context{}; mbedtls_sha1_context m_sha1_context{}; diff --git a/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPProto.h b/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPProto.h index a397431e3d..aca2176e6d 100644 --- a/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPProto.h +++ b/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPProto.h @@ -7,9 +7,8 @@ #include #include -#include - #include "Common/CommonTypes.h" +#include "Common/Hash.h" namespace ciface::DualShockUDPClient::Proto { @@ -217,11 +216,6 @@ struct FromClient }; } // namespace MessageType -static inline u32 CRC32(const void* buffer, unsigned length) -{ - return crc32(crc32(0L, Z_NULL, 0), static_cast(buffer), length); -} - template struct Message { @@ -236,7 +230,11 @@ struct Message m_message.message_type = MsgType::TYPE; } - void Finish() { m_message.header.crc32 = CRC32(&m_message, sizeof(m_message)); } + void Finish() + { + m_message.header.crc32 = + Common::ComputeCRC32(reinterpret_cast(&m_message), sizeof(m_message)); + } template std::optional CheckAndCastTo() @@ -244,7 +242,8 @@ struct Message const u32 crc32_in_header = m_message.header.crc32; // zero out the crc32 in the packet once we got it since that's whats needed for calculation m_message.header.crc32 = 0; - const u32 crc32_calculated = CRC32(&m_message, sizeof(ToMsgType)); + const u32 crc32_calculated = + Common::ComputeCRC32(reinterpret_cast(&m_message), sizeof(ToMsgType)); if (crc32_in_header != crc32_calculated) { NOTICE_LOG_FMT( From 301bc49efe23d93cdc870508b28a6bc96be645d5 Mon Sep 17 00:00:00 2001 From: Pokechu22 Date: Fri, 10 Dec 2021 15:21:46 -0800 Subject: [PATCH 3/4] Common: Remove MD5.h It uses DiscIO, and Common shouldn't depend on DiscIO. Instead, put this code in NetPlayClient.cpp. --- Source/Core/Common/CMakeLists.txt | 2 -- Source/Core/Common/MD5.cpp | 54 ------------------------------ Source/Core/Common/MD5.h | 12 ------- Source/Core/Core/NetPlayClient.cpp | 42 +++++++++++++++++++++-- Source/Core/DolphinLib.props | 2 -- 5 files changed, 40 insertions(+), 72 deletions(-) delete mode 100644 Source/Core/Common/MD5.cpp delete mode 100644 Source/Core/Common/MD5.h diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index a2156c5b5d..5314f119f0 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -81,8 +81,6 @@ add_library(common MathUtil.h Matrix.cpp Matrix.h - MD5.cpp - MD5.h MemArena.h MemoryUtil.cpp MemoryUtil.h diff --git a/Source/Core/Common/MD5.cpp b/Source/Core/Common/MD5.cpp deleted file mode 100644 index 976d80575b..0000000000 --- a/Source/Core/Common/MD5.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2016 Dolphin Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#include "Common/MD5.h" - -#include -#include -#include -#include - -#include - -#include "Common/StringUtil.h" -#include "DiscIO/Blob.h" - -namespace MD5 -{ -std::string MD5Sum(const std::string& file_path, std::function report_progress) -{ - std::string output_string; - std::vector data(8 * 1024 * 1024); - u64 read_offset = 0; - mbedtls_md5_context ctx; - - std::unique_ptr file(DiscIO::CreateBlobReader(file_path)); - u64 game_size = file->GetDataSize(); - - mbedtls_md5_starts_ret(&ctx); - - while (read_offset < game_size) - { - size_t read_size = std::min(static_cast(data.size()), game_size - read_offset); - if (!file->Read(read_offset, read_size, data.data())) - return output_string; - - mbedtls_md5_update_ret(&ctx, data.data(), read_size); - read_offset += read_size; - - int progress = - static_cast(static_cast(read_offset) / static_cast(game_size) * 100); - if (!report_progress(progress)) - return output_string; - } - - std::array output; - mbedtls_md5_finish_ret(&ctx, output.data()); - - // Convert to hex - for (u8 n : output) - output_string += fmt::format("{:02x}", n); - - return output_string; -} -} // namespace MD5 diff --git a/Source/Core/Common/MD5.h b/Source/Core/Common/MD5.h deleted file mode 100644 index 58eabc6816..0000000000 --- a/Source/Core/Common/MD5.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2016 Dolphin Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include -#include - -namespace MD5 -{ -std::string MD5Sum(const std::string& file_name, std::function progress); -} diff --git a/Source/Core/Core/NetPlayClient.cpp b/Source/Core/Core/NetPlayClient.cpp index 45782dfe21..115b101499 100644 --- a/Source/Core/Core/NetPlayClient.cpp +++ b/Source/Core/Core/NetPlayClient.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -23,7 +24,6 @@ #include "Common/ENetUtil.h" #include "Common/FileUtil.h" #include "Common/Logging/Log.h" -#include "Common/MD5.h" #include "Common/MsgHandler.h" #include "Common/NandPaths.h" #include "Common/QoSSession.h" @@ -62,6 +62,7 @@ #include "Core/NetPlayCommon.h" #include "Core/PowerPC/PowerPC.h" #include "Core/SyncIdentifier.h" +#include "DiscIO/Blob.h" #include "InputCommon/ControllerEmu/ControlGroup/Attachments.h" #include "InputCommon/GCAdapter.h" @@ -2462,6 +2463,43 @@ bool NetPlayClient::DoAllPlayersHaveGame() }); } +static std::string MD5Sum(const std::string& file_path, std::function report_progress) +{ + std::string output_string; + std::vector data(8 * 1024 * 1024); + u64 read_offset = 0; + mbedtls_md5_context ctx; + + std::unique_ptr file(DiscIO::CreateBlobReader(file_path)); + u64 game_size = file->GetDataSize(); + + mbedtls_md5_starts_ret(&ctx); + + while (read_offset < game_size) + { + size_t read_size = std::min(static_cast(data.size()), game_size - read_offset); + if (!file->Read(read_offset, read_size, data.data())) + return output_string; + + mbedtls_md5_update_ret(&ctx, data.data(), read_size); + read_offset += read_size; + + int progress = + static_cast(static_cast(read_offset) / static_cast(game_size) * 100); + if (!report_progress(progress)) + return output_string; + } + + std::array output; + mbedtls_md5_finish_ret(&ctx, output.data()); + + // Convert to hex + for (u8 n : output) + output_string += fmt::format("{:02x}", n); + + return output_string; +} + void NetPlayClient::ComputeMD5(const SyncIdentifier& sync_identifier) { if (m_should_compute_MD5) @@ -2488,7 +2526,7 @@ void NetPlayClient::ComputeMD5(const SyncIdentifier& sync_identifier) if (m_MD5_thread.joinable()) m_MD5_thread.join(); m_MD5_thread = std::thread([this, file]() { - std::string sum = MD5::MD5Sum(file, [&](int progress) { + std::string sum = MD5Sum(file, [&](int progress) { sf::Packet packet; packet << MessageID::MD5Progress; packet << progress; diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index 6c267853df..60a4e2299d 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -123,7 +123,6 @@ - @@ -723,7 +722,6 @@ - From 3d5b46615cfc6d238075a9a1666c29f6f54decbe Mon Sep 17 00:00:00 2001 From: Pokechu22 Date: Sat, 18 Dec 2021 12:46:47 -0800 Subject: [PATCH 4/4] NetPlayClient: Use fmt::join for MD5Sum --- Source/Core/Core/NetPlayClient.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Source/Core/Core/NetPlayClient.cpp b/Source/Core/Core/NetPlayClient.cpp index 115b101499..ff9dca8c22 100644 --- a/Source/Core/Core/NetPlayClient.cpp +++ b/Source/Core/Core/NetPlayClient.cpp @@ -2465,7 +2465,6 @@ bool NetPlayClient::DoAllPlayersHaveGame() static std::string MD5Sum(const std::string& file_path, std::function report_progress) { - std::string output_string; std::vector data(8 * 1024 * 1024); u64 read_offset = 0; mbedtls_md5_context ctx; @@ -2479,7 +2478,7 @@ static std::string MD5Sum(const std::string& file_path, std::function { size_t read_size = std::min(static_cast(data.size()), game_size - read_offset); if (!file->Read(read_offset, read_size, data.data())) - return output_string; + return ""; mbedtls_md5_update_ret(&ctx, data.data(), read_size); read_offset += read_size; @@ -2487,17 +2486,14 @@ static std::string MD5Sum(const std::string& file_path, std::function int progress = static_cast(static_cast(read_offset) / static_cast(game_size) * 100); if (!report_progress(progress)) - return output_string; + return ""; } std::array output; mbedtls_md5_finish_ret(&ctx, output.data()); // Convert to hex - for (u8 n : output) - output_string += fmt::format("{:02x}", n); - - return output_string; + return fmt::format("{:02x}", fmt::join(output, "")); } void NetPlayClient::ComputeMD5(const SyncIdentifier& sync_identifier)