From 3f6a976e0f03ecc631318bb00952e32214e11ee7 Mon Sep 17 00:00:00 2001 From: Sketch <75850871+SketchMaster2001@users.noreply.github.com> Date: Tue, 5 Sep 2023 20:56:53 -0400 Subject: [PATCH] IOS/KD: Implement Download Scheduler --- Source/Core/Core/IOS/Network/KD/NWC24Config.h | 1 + Source/Core/Core/IOS/Network/KD/NWC24DL.cpp | 72 ++++++++++- Source/Core/Core/IOS/Network/KD/NWC24DL.h | 25 +++- .../Core/Core/IOS/Network/KD/NetKDRequest.cpp | 121 +++++++++++++++++- .../Core/Core/IOS/Network/KD/NetKDRequest.h | 2 + Source/Core/Core/IOS/Network/KD/NetKDTime.h | 8 +- 6 files changed, 215 insertions(+), 14 deletions(-) diff --git a/Source/Core/Core/IOS/Network/KD/NWC24Config.h b/Source/Core/Core/IOS/Network/KD/NWC24Config.h index 2b05d6dfd7..26d37009ff 100644 --- a/Source/Core/Core/IOS/Network/KD/NWC24Config.h +++ b/Source/Core/Core/IOS/Network/KD/NWC24Config.h @@ -32,6 +32,7 @@ enum ErrorCode : s32 WC24_ERR_ID_NONEXISTANCE = -34, WC24_ERR_ID_GENERATED = -35, WC24_ERR_ID_REGISTERED = -36, + WC24_ERR_DISABLED = -39, WC24_ERR_ID_NOT_REGISTERED = -44, }; diff --git a/Source/Core/Core/IOS/Network/KD/NWC24DL.cpp b/Source/Core/Core/IOS/Network/KD/NWC24DL.cpp index e61f3abbf1..34c5e4d1d1 100644 --- a/Source/Core/Core/IOS/Network/KD/NWC24DL.cpp +++ b/Source/Core/Core/IOS/Network/KD/NWC24DL.cpp @@ -59,7 +59,7 @@ void NWC24Dl::WriteDlList() const ERROR_LOG_FMT(IOS_WC24, "Failed to open or write WC24 DL list file"); } -bool NWC24Dl::DoesEntryExist(u16 entry_index) +bool NWC24Dl::DoesEntryExist(u16 entry_index) const { return m_data.entries[entry_index].low_title_id != 0; } @@ -125,6 +125,76 @@ bool NWC24Dl::IsRSASigned(u16 entry_index) const return !Common::ExtractBit(Common::swap32(m_data.entries[entry_index].flags), 2); } +bool NWC24Dl::SkipSchedulerDownload(u16 entry_index) const +{ + // Some entries can be set to not be downloaded by the scheduler. + return !!Common::ExtractBit(Common::swap32(m_data.entries[entry_index].flags), 5); +} + +bool NWC24Dl::HasSubtask(u16 entry_index) const +{ + switch (m_data.entries[entry_index].subtask_type) + { + case 1: + case 2: + case 3: + case 4: + return true; + default: + return false; + } +} + +bool NWC24Dl::IsSubtaskDownloadDisabled(u16 entry_index) const +{ + return !!Common::ExtractBit(Common::swap16(m_data.entries[entry_index].subtask_flags), 9); +} + +bool NWC24Dl::IsValidSubtask(u16 entry_index, u8 subtask_id) const +{ + return !!Common::ExtractBit(m_data.entries[entry_index].subtask_bitmask, subtask_id); +} + +u64 NWC24Dl::GetNextDownloadTime(u16 record_index) const +{ + // Timestamps are stored as a UNIX timestamp but in minutes. We want seconds. + return Common::swap32(m_data.records[record_index].next_dl_timestamp) * SECONDS_PER_MINUTE; +} + +u64 NWC24Dl::GetRetryTime(u16 entry_index) const +{ + const u64 retry_time = Common::swap16(m_data.entries[entry_index].retry_frequency); + if (retry_time == 0) + { + return MINUTES_PER_DAY * SECONDS_PER_MINUTE; + } + return retry_time * SECONDS_PER_MINUTE; +} + +u64 NWC24Dl::GetDownloadMargin(u16 entry_index) const +{ + return Common::swap16(m_data.entries[entry_index].dl_margin) * SECONDS_PER_MINUTE; +} + +void NWC24Dl::SetNextDownloadTime(u16 record_index, u64 value, std::optional subtask_id) +{ + if (subtask_id) + { + m_data.entries[record_index].subtask_timestamps[*subtask_id] = + Common::swap32(static_cast(value / SECONDS_PER_MINUTE)); + } + + m_data.records[record_index].next_dl_timestamp = + Common::swap32(static_cast(value / SECONDS_PER_MINUTE)); +} + +u64 NWC24Dl::GetLastSubtaskDownloadTime(u16 entry_index, u8 subtask_id) const +{ + return Common::swap32(m_data.entries[entry_index].subtask_timestamps[subtask_id]) * + SECONDS_PER_MINUTE + + Common::swap32(m_data.entries[entry_index].server_dl_interval) * SECONDS_PER_MINUTE; +} + u32 NWC24Dl::Magic() const { return Common::swap32(m_data.header.magic); diff --git a/Source/Core/Core/IOS/Network/KD/NWC24DL.h b/Source/Core/Core/IOS/Network/KD/NWC24DL.h index e5427ea2ea..d6cbb8e9c8 100644 --- a/Source/Core/Core/IOS/Network/KD/NWC24DL.h +++ b/Source/Core/Core/IOS/Network/KD/NWC24DL.h @@ -27,14 +27,24 @@ public: s32 CheckNwc24DlList() const; - bool DoesEntryExist(u16 entry_index); + bool DoesEntryExist(u16 entry_index) const; bool IsEncrypted(u16 entry_index) const; bool IsRSASigned(u16 entry_index) const; + bool SkipSchedulerDownload(u16 entry_index) const; + bool HasSubtask(u16 entry_index) const; + bool IsSubtaskDownloadDisabled(u16 entry_index) const; + bool IsValidSubtask(u16 entry_index, u8 subtask_id) const; std::string GetVFFContentName(u16 entry_index, std::optional subtask_id) const; std::string GetDownloadURL(u16 entry_index, std::optional subtask_id) const; std::string GetVFFPath(u16 entry_index) const; WC24PubkMod GetWC24PubkMod(u16 entry_index) const; + u64 GetNextDownloadTime(u16 record_index) const; + u64 GetDownloadMargin(u16 entry_index) const; + void SetNextDownloadTime(u16 record_index, u64 value, std::optional subtask_id); + u64 GetRetryTime(u16 entry_index) const; + u64 GetLastSubtaskDownloadTime(u16 entry_index, u8 subtask_id) const; + u32 Magic() const; void SetMagic(u32 magic); @@ -46,10 +56,12 @@ public: private: static constexpr u32 DL_LIST_MAGIC = 0x5763446C; // WcDl static constexpr u32 MAX_SUBENTRIES = 32; + static constexpr u64 SECONDS_PER_MINUTE = 60; + static constexpr u64 MINUTES_PER_DAY = 1440; enum EntryType : u8 { - UNK = 1, + SUBTASK = 1, MAIL, CHANNEL_CONTENT, UNUSED = 0xff @@ -91,15 +103,14 @@ private: u16 padding1; u16 remaining_downloads; u16 error_count; - u16 dl_frequency; - u16 dl_frequency_when_err; + u16 dl_margin; + u16 retry_frequency; s32 error_code; u8 subtask_id; u8 subtask_type; - u8 subtask_flags; - u8 padding2; + u16 subtask_flags; u32 subtask_bitmask; - s32 unknown2; + u32 server_dl_interval; u32 dl_timestamp; // Last DL time u32 subtask_timestamps[32]; char dl_url[236]; diff --git a/Source/Core/Core/IOS/Network/KD/NetKDRequest.cpp b/Source/Core/Core/IOS/Network/KD/NetKDRequest.cpp index 6bf7d6b766..833bc5707f 100644 --- a/Source/Core/Core/IOS/Network/KD/NetKDRequest.cpp +++ b/Source/Core/Core/IOS/Network/KD/NetKDRequest.cpp @@ -23,6 +23,7 @@ #include "Core/CommonTitles.h" #include "Core/HW/Memmap.h" #include "Core/IOS/FS/FileSystem.h" +#include "Core/IOS/Network/KD/NetKDTime.h" #include "Core/IOS/Network/KD/VFF/VFFUtil.h" #include "Core/IOS/Network/Socket.h" #include "Core/IOS/Uids.h" @@ -239,8 +240,20 @@ void NetKDRequestDevice::SchedulerWorker(const SchedulerEvent event) { if (event == SchedulerEvent::Download) { - // TODO: Implement downloader part of scheduler - return; + u16 entry_index{}; + std::optional subtask_id{}; + NWC24::ErrorCode code = DetermineDownloadTask(&entry_index, &subtask_id); + if (code != NWC24::WC24_OK) + { + LogError(ErrorType::KD_Download, code); + return; + } + + code = KDDownload(entry_index, subtask_id); + if (code != NWC24::WC24_OK) + { + LogError(ErrorType::KD_Download, code); + } } else { @@ -400,9 +413,109 @@ NWC24::ErrorCode NetKDRequestDevice::KDCheckMail(u32* mail_flag, u32* interval) return NWC24::WC24_OK; } +NWC24::ErrorCode NetKDRequestDevice::DetermineDownloadTask(u16* entry_index, + std::optional* subtask_id) const +{ + // As the scheduler does not tell us which entry to download, we must determine that. + // A correct entry is one that hasn't been downloaded the longest compared to other entries. + // We first need current UTC. + const auto time_device = + std::static_pointer_cast(GetIOS()->GetDeviceByName("/dev/net/kd/time")); + const u64 current_utc = time_device->GetAdjustedUTC(); + u64 lowest_timestamp = std::numeric_limits::max(); + + for (u16 i = 0; i < static_cast(NWC24::NWC24Dl::MAX_ENTRIES); i++) + { + if (!m_dl_list.DoesEntryExist(i)) + continue; + + if (m_dl_list.SkipSchedulerDownload(i)) + continue; + + const u64 next_dl_time = m_dl_list.GetNextDownloadTime(i); + + // First determine if UTC is greater than the next download time. + if (current_utc < next_dl_time) + continue; + + // If this task's next download time is less than the lowest_timestamp, this is the task we + // want. However, we must determine if this has a subtask and wants to be downloaded. + if (next_dl_time < lowest_timestamp) + { + if (m_dl_list.HasSubtask(i)) + { + NWC24::ErrorCode code = DetermineSubtask(i, subtask_id); + if (code != NWC24::WC24_OK) + { + // No valid subtask found or downloading is disabled. + continue; + } + } + + lowest_timestamp = next_dl_time; + *entry_index = i; + } + } + + // Determine if we actually found an entry to download. + if (lowest_timestamp == std::numeric_limits::max()) + return NWC24::WC24_ERR_NOT_FOUND; + + return NWC24::WC24_OK; +} + +NWC24::ErrorCode NetKDRequestDevice::DetermineSubtask(u16 entry_index, + std::optional* subtask_id) const +{ + // Before we do anything, determine if this task wants to be downloaded + if (m_dl_list.IsSubtaskDownloadDisabled(entry_index)) + return NWC24::WC24_ERR_DISABLED; + + const auto time_device = + std::static_pointer_cast(GetIOS()->GetDeviceByName("/dev/net/kd/time")); + const u64 current_utc = time_device->GetAdjustedUTC(); + for (u8 i = 0; i < 32; i++) + { + if (!m_dl_list.IsValidSubtask(entry_index, i)) + continue; + + // Unlike DetermineDownloadTask, DetermineSubtask gets the first download time lower than UTC. + const u64 last_download_time = m_dl_list.GetLastSubtaskDownloadTime(entry_index, i); + if (last_download_time < current_utc) + { + *subtask_id = i; + return NWC24::WC24_OK; + } + } + + return NWC24::WC24_ERR_INVALID_VALUE; +} + NWC24::ErrorCode NetKDRequestDevice::KDDownload(const u16 entry_index, const std::optional subtask_id) { + bool success = false; + Common::ScopeGuard state_guard([&] { + const auto time_device = + std::static_pointer_cast(GetIOS()->GetDeviceByName("/dev/net/kd/time")); + const u64 current_utc = time_device->GetAdjustedUTC(); + if (success) + { + // Set the next download time to the dl_margin + m_dl_list.SetNextDownloadTime( + entry_index, current_utc + m_dl_list.GetDownloadMargin(entry_index), subtask_id); + } + else + { + // Else set it to the retry margin + m_dl_list.SetNextDownloadTime(entry_index, current_utc + m_dl_list.GetRetryTime(entry_index), + subtask_id); + } + + // Finally flush + m_dl_list.WriteDlList(); + }); + std::vector file_data; // Content metadata @@ -492,8 +605,12 @@ NWC24::ErrorCode NetKDRequestDevice::KDDownload(const u16 entry_index, m_ios.GetFS(), file_data); if (reply != NWC24::WC24_OK) + { LogError(ErrorType::KD_Download, reply); + return reply; + } + success = true; return reply; } diff --git a/Source/Core/Core/IOS/Network/KD/NetKDRequest.h b/Source/Core/Core/IOS/Network/KD/NetKDRequest.h index 4f7a780359..99130cec6b 100644 --- a/Source/Core/Core/IOS/Network/KD/NetKDRequest.h +++ b/Source/Core/Core/IOS/Network/KD/NetKDRequest.h @@ -85,6 +85,8 @@ private: void LogError(ErrorType error_type, s32 error_code); void SchedulerTimer(); void SchedulerWorker(SchedulerEvent event); + NWC24::ErrorCode DetermineDownloadTask(u16* entry_index, std::optional* subtask_id) const; + NWC24::ErrorCode DetermineSubtask(u16 entry_index, std::optional* subtask_id) const; static std::string GetValueFromCGIResponse(const std::string& response, const std::string& key); static constexpr std::array MAIL_CHECK_KEY = {0xce, 0x4c, 0xf2, 0x9a, 0x3d, 0x6b, 0xe1, diff --git a/Source/Core/Core/IOS/Network/KD/NetKDTime.h b/Source/Core/Core/IOS/Network/KD/NetKDTime.h index 025594c921..4f24c8b36a 100644 --- a/Source/Core/Core/IOS/Network/KD/NetKDTime.h +++ b/Source/Core/Core/IOS/Network/KD/NetKDTime.h @@ -18,15 +18,15 @@ public: std::optional IOCtl(const IOCtlRequest& request) override; + // Returns seconds since Wii epoch + // +/- any bias set from IOCTL_NW24_SET_UNIVERSAL_TIME + u64 GetAdjustedUTC() const; + private: // TODO: depending on CEXIIPL is a hack which I don't feel like // removing because the function itself is pretty hackish; // wait until I re-port my netplay rewrite - // Returns seconds since Wii epoch - // +/- any bias set from IOCTL_NW24_SET_UNIVERSAL_TIME - u64 GetAdjustedUTC() const; - // Store the difference between what the Wii thinks is UTC and // what the host OS thinks void SetAdjustedUTC(u64 wii_utc);