mirror of
https://github.com/dolphin-emu/dolphin.git
synced 2024-11-15 05:47:56 -07:00
c0eb95481f
This improves the speed of verifying Wii WIA/RVZ files. For me, the verification speed for LZMA2-compressed files has gone from 11-12 MiB/s to 13-14 MiB/s. One thing VolumeVerifier does to achieve parallelism is to compute hashes for one chunk of data while reading the next chunk of data. In master, when reading data from a Wii partition, each such chunk is 32 KiB. This is normally fine, but with WIA and RVZ it leads to rather lopsided read times (without the compute times being lopsided): The first 32 KiB of each 2 MiB takes a long time to read, and the remaining part of the 2 MiB can be read nearly instantly. (The WIA/RVZ code has to read the entire 2 MiB in order to compute hashes which appear at the beginning of the 2 MiB, and then caches the result afterwards.) This leads to us at times not doing much reading and at other times not doing much computation. To improve this, this change makes us use 2 MiB chunks instead of 32 KiB chunks when reading from Wii partitions. (block = 32 KiB, group = 2 MiB)
207 lines
4.6 KiB
C++
207 lines
4.6 KiB
C++
// Copyright 2019 Dolphin Emulator Project
|
|
// Licensed under GPLv2+
|
|
// Refer to the license.txt file included.
|
|
|
|
#pragma once
|
|
|
|
#include <future>
|
|
#include <map>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <mbedtls/md5.h>
|
|
#include <mbedtls/sha1.h>
|
|
|
|
#include "Common/CommonTypes.h"
|
|
#include "Core/IOS/ES/Formats.h"
|
|
#include "DiscIO/DiscScrubber.h"
|
|
#include "DiscIO/Volume.h"
|
|
|
|
// To be used as follows:
|
|
//
|
|
// VolumeVerifier verifier(volume, redump_verification, hashes_to_calculate);
|
|
// verifier.Start();
|
|
// while (verifier.GetBytesProcessed() != verifier.GetTotalBytes())
|
|
// verifier.Process();
|
|
// verifier.Finish();
|
|
// auto result = verifier.GetResult();
|
|
//
|
|
// Start, Process and Finish may take some time to run.
|
|
//
|
|
// GetResult() can be called before the processing is finished, but the result will be incomplete.
|
|
|
|
namespace DiscIO
|
|
{
|
|
template <typename T>
|
|
struct Hashes
|
|
{
|
|
T crc32;
|
|
T md5;
|
|
T sha1;
|
|
};
|
|
|
|
class RedumpVerifier final
|
|
{
|
|
public:
|
|
enum class Status
|
|
{
|
|
Unknown,
|
|
GoodDump,
|
|
BadDump,
|
|
Error,
|
|
};
|
|
|
|
struct Result
|
|
{
|
|
Status status = Status::Unknown;
|
|
std::string message;
|
|
};
|
|
|
|
void Start(const Volume& volume);
|
|
Result Finish(const Hashes<std::vector<u8>>& hashes);
|
|
|
|
private:
|
|
enum class DownloadStatus
|
|
{
|
|
NotAttempted,
|
|
Success,
|
|
Fail,
|
|
FailButOldCacheAvailable,
|
|
SystemNotAvailable,
|
|
};
|
|
|
|
struct DownloadState
|
|
{
|
|
std::mutex mutex;
|
|
DownloadStatus status = DownloadStatus::NotAttempted;
|
|
};
|
|
|
|
struct PotentialMatch
|
|
{
|
|
u64 size;
|
|
Hashes<std::vector<u8>> hashes;
|
|
};
|
|
|
|
static DownloadStatus DownloadDatfile(const std::string& system, DownloadStatus old_status);
|
|
static std::vector<u8> ReadDatfile(const std::string& system);
|
|
std::vector<PotentialMatch> ScanDatfile(const std::vector<u8>& data, const std::string& system);
|
|
|
|
std::string m_game_id;
|
|
u16 m_revision;
|
|
u8 m_disc_number;
|
|
u64 m_size;
|
|
|
|
std::future<std::vector<PotentialMatch>> m_future;
|
|
Result m_result;
|
|
|
|
static DownloadState m_gc_download_state;
|
|
static DownloadState m_wii_download_state;
|
|
};
|
|
|
|
class VolumeVerifier final
|
|
{
|
|
public:
|
|
enum class Severity
|
|
{
|
|
None, // Only used internally
|
|
Low,
|
|
Medium,
|
|
High,
|
|
};
|
|
|
|
struct Problem
|
|
{
|
|
Severity severity;
|
|
std::string text;
|
|
};
|
|
|
|
struct Result
|
|
{
|
|
Hashes<std::vector<u8>> hashes;
|
|
std::string summary_text;
|
|
std::vector<Problem> problems;
|
|
RedumpVerifier::Result redump;
|
|
};
|
|
|
|
VolumeVerifier(const Volume& volume, bool redump_verification, Hashes<bool> hashes_to_calculate);
|
|
~VolumeVerifier();
|
|
|
|
void Start();
|
|
void Process();
|
|
u64 GetBytesProcessed() const;
|
|
u64 GetTotalBytes() const;
|
|
void Finish();
|
|
const Result& GetResult() const;
|
|
|
|
private:
|
|
struct GroupToVerify
|
|
{
|
|
Partition partition;
|
|
u64 offset;
|
|
size_t block_index_start;
|
|
size_t block_index_end;
|
|
};
|
|
|
|
std::vector<Partition> CheckPartitions();
|
|
bool CheckPartition(const Partition& partition); // Returns false if partition should be ignored
|
|
std::string GetPartitionName(std::optional<u32> type) const;
|
|
bool IsDebugSigned() const;
|
|
bool ShouldHaveChannelPartition() const;
|
|
bool ShouldHaveInstallPartition() const;
|
|
bool ShouldHaveMasterpiecePartitions() const;
|
|
bool ShouldBeDualLayer() const;
|
|
void CheckVolumeSize();
|
|
void CheckMisc();
|
|
void CheckSuperPaperMario();
|
|
void SetUpHashing();
|
|
void WaitForAsyncOperations() const;
|
|
bool ReadChunkAndWaitForAsyncOperations(u64 bytes_to_read);
|
|
|
|
void AddProblem(Severity severity, std::string text);
|
|
|
|
const Volume& m_volume;
|
|
Result m_result;
|
|
bool m_is_tgc = false;
|
|
bool m_is_datel = false;
|
|
bool m_is_not_retail = false;
|
|
|
|
bool m_redump_verification;
|
|
RedumpVerifier m_redump_verifier;
|
|
|
|
bool m_read_errors_occurred = false;
|
|
|
|
Hashes<bool> m_hashes_to_calculate{};
|
|
bool m_calculating_any_hash = false;
|
|
unsigned long m_crc32_context = 0;
|
|
mbedtls_md5_context m_md5_context;
|
|
mbedtls_sha1_context m_sha1_context;
|
|
|
|
u64 m_excess_bytes = 0;
|
|
std::vector<u8> m_data;
|
|
std::future<void> m_crc32_future;
|
|
std::future<void> m_md5_future;
|
|
std::future<void> m_sha1_future;
|
|
std::future<void> m_content_future;
|
|
std::future<void> m_group_future;
|
|
|
|
DiscScrubber m_scrubber;
|
|
IOS::ES::TicketReader m_ticket;
|
|
std::vector<u64> m_content_offsets;
|
|
u16 m_content_index = 0;
|
|
std::vector<GroupToVerify> m_groups;
|
|
size_t m_group_index = 0; // Index in m_groups, not index in a specific partition
|
|
std::map<Partition, size_t> m_block_errors;
|
|
std::map<Partition, size_t> m_unused_block_errors;
|
|
|
|
u64 m_biggest_referenced_offset = 0;
|
|
u64 m_biggest_verified_offset = 0;
|
|
|
|
bool m_started = false;
|
|
bool m_done = false;
|
|
u64 m_progress = 0;
|
|
u64 m_max_progress = 0;
|
|
};
|
|
|
|
} // namespace DiscIO
|