diff --git a/.gitmodules b/.gitmodules index a232b78dcf..189633dc58 100644 --- a/.gitmodules +++ b/.gitmodules @@ -84,6 +84,9 @@ [submodule "Externals/Vulkan-Headers"] path = Externals/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "Externals/watcher/watcher"] + path = Externals/watcher/watcher + url = https://github.com/e-dant/watcher.git [submodule "Externals/SFML/SFML"] path = Externals/SFML/SFML url = https://github.com/SFML/SFML.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b4202975d..549007ddd6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -784,6 +784,8 @@ if (USE_RETRO_ACHIEVEMENTS) add_subdirectory(Externals/rcheevos) endif() +add_subdirectory(Externals/watcher) + ######################################## # Pre-build events: Define configuration variables and write SCM info header # diff --git a/Externals/watcher/CMakeLists.txt b/Externals/watcher/CMakeLists.txt new file mode 100644 index 0000000000..097f16d647 --- /dev/null +++ b/Externals/watcher/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(watcher INTERFACE IMPORTED GLOBAL) +set_target_properties(watcher PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/watcher/include +) diff --git a/Externals/watcher/watcher b/Externals/watcher/watcher new file mode 160000 index 0000000000..b03bdcfc11 --- /dev/null +++ b/Externals/watcher/watcher @@ -0,0 +1 @@ +Subproject commit b03bdcfc11549df595b77239cefe2643943a3e2f diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index bbc3721a19..6d1b6908f3 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -64,6 +64,8 @@ add_library(common FatFsUtil.h FileSearch.cpp FileSearch.h + FilesystemWatcher.cpp + FilesystemWatcher.h FileUtil.cpp FileUtil.h FixedSizeQueue.h @@ -184,6 +186,7 @@ PRIVATE FatFs Iconv::Iconv spng::spng + watcher ${VTUNE_LIBRARIES} ) diff --git a/Source/Core/Common/FilesystemWatcher.cpp b/Source/Core/Common/FilesystemWatcher.cpp new file mode 100644 index 0000000000..646e2467ed --- /dev/null +++ b/Source/Core/Common/FilesystemWatcher.cpp @@ -0,0 +1,67 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "Common/FilesystemWatcher.h" + +#include + +#include "Common/Logging/Log.h" +#include "Common/StringUtil.h" + +namespace Common +{ +FilesystemWatcher::FilesystemWatcher() = default; +FilesystemWatcher::~FilesystemWatcher() = default; + +void FilesystemWatcher::Watch(const std::string& path) +{ + const auto [iter, inserted] = m_watched_paths.try_emplace(path, nullptr); + if (inserted) + { + iter->second = std::make_unique(path, [this](wtr::event e) { + const auto watched_path = PathToString(e.path_name); + if (e.path_type == wtr::event::path_type::watcher) + { + if (watched_path.starts_with('e')) + ERROR_LOG_FMT(COMMON, "Filesystem watcher: '{}'", watched_path); + else if (watched_path.starts_with('w')) + WARN_LOG_FMT(COMMON, "Filesystem watcher: '{}'", watched_path); + return; + } + + if (e.effect_type == wtr::event::effect_type::create) + { + const auto path = WithUnifiedPathSeparators(watched_path); + PathAdded(path); + } + else if (e.effect_type == wtr::event::effect_type::modify) + { + const auto path = WithUnifiedPathSeparators(watched_path); + PathModified(path); + } + else if (e.effect_type == wtr::event::effect_type::rename) + { + if (!e.associated) + { + WARN_LOG_FMT(COMMON, "Rename on path '{}' seen without association!", watched_path); + return; + } + + const auto old_path = WithUnifiedPathSeparators(watched_path); + const auto new_path = WithUnifiedPathSeparators(PathToString(e.associated->path_name)); + PathRenamed(old_path, new_path); + } + else if (e.effect_type == wtr::event::effect_type::destroy) + { + const auto path = WithUnifiedPathSeparators(watched_path); + PathDeleted(path); + } + }); + } +} + +void FilesystemWatcher::Unwatch(const std::string& path) +{ + m_watched_paths.erase(path); +} +} // namespace Common diff --git a/Source/Core/Common/FilesystemWatcher.h b/Source/Core/Common/FilesystemWatcher.h new file mode 100644 index 0000000000..ad1c822e2b --- /dev/null +++ b/Source/Core/Common/FilesystemWatcher.h @@ -0,0 +1,47 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +namespace wtr +{ +inline namespace watcher +{ +class watch; +} +} // namespace wtr + +namespace Common +{ +// A class that can watch a path and receive callbacks +// when files or directories underneath that path receive events +class FilesystemWatcher +{ +public: + FilesystemWatcher(); + virtual ~FilesystemWatcher(); + + void Watch(const std::string& path); + void Unwatch(const std::string& path); + +private: + // A new file or folder was added to one of the watched paths + virtual void PathAdded(std::string_view path) {} + + // A file or folder was modified in one of the watched paths + virtual void PathModified(std::string_view path) {} + + // A file or folder was renamed in one of the watched paths + virtual void PathRenamed(std::string_view old_path, std::string_view new_path) {} + + // A file or folder was deleted in one of the watched paths + virtual void PathDeleted(std::string_view path) {} + + std::map> m_watched_paths; +}; +} // namespace Common diff --git a/Source/Core/Core/Core.cpp b/Source/Core/Core/Core.cpp index e80e382930..7de860e59c 100644 --- a/Source/Core/Core/Core.cpp +++ b/Source/Core/Core/Core.cpp @@ -82,7 +82,6 @@ #include "InputCommon/ControllerInterface/ControllerInterface.h" #include "InputCommon/GCAdapter.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" #include "VideoCommon/AsyncRequests.h" #include "VideoCommon/Fifo.h" #include "VideoCommon/FrameDumper.h" @@ -528,9 +527,6 @@ static void EmuThread(Core::System& system, std::unique_ptr boot FreeLook::LoadInputConfig(); - system.GetCustomAssetLoader().Init(); - Common::ScopeGuard asset_loader_guard([&system] { system.GetCustomAssetLoader().Shutdown(); }); - system.GetMovie().Init(*boot); Common::ScopeGuard movie_guard([&system] { system.GetMovie().Shutdown(); }); diff --git a/Source/Core/Core/System.cpp b/Source/Core/Core/System.cpp index d52975fe1b..0170c5fd11 100644 --- a/Source/Core/Core/System.cpp +++ b/Source/Core/Core/System.cpp @@ -33,7 +33,7 @@ #include "IOS/USB/Emulated/Infinity.h" #include "IOS/USB/Emulated/Skylanders/Skylander.h" #include "IOS/USB/USBScanner.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/CommandProcessor.h" #include "VideoCommon/Fifo.h" #include "VideoCommon/GeometryShaderManager.h" @@ -96,7 +96,7 @@ struct System::Impl VideoInterface::VideoInterfaceManager m_video_interface; Interpreter m_interpreter; JitInterface m_jit_interface; - VideoCommon::CustomAssetLoader m_custom_asset_loader; + VideoCommon::CustomResourceManager m_custom_resource_manager; FifoPlayer m_fifo_player; FifoRecorder m_fifo_recorder; Movie::MovieManager m_movie; @@ -335,8 +335,8 @@ VideoInterface::VideoInterfaceManager& System::GetVideoInterface() const return m_impl->m_video_interface; } -VideoCommon::CustomAssetLoader& System::GetCustomAssetLoader() const +VideoCommon::CustomResourceManager& System::GetCustomResourceManager() const { - return m_impl->m_custom_asset_loader; + return m_impl->m_custom_resource_manager; } } // namespace Core diff --git a/Source/Core/Core/System.h b/Source/Core/Core/System.h index 67069c5e22..348f4fcb4a 100644 --- a/Source/Core/Core/System.h +++ b/Source/Core/Core/System.h @@ -108,8 +108,8 @@ class SystemTimersManager; } namespace VideoCommon { -class CustomAssetLoader; -} +class CustomResourceManager; +} // namespace VideoCommon namespace VideoInterface { class VideoInterfaceManager; @@ -197,7 +197,7 @@ public: VertexShaderManager& GetVertexShaderManager() const; XFStateManager& GetXFStateManager() const; VideoInterface::VideoInterfaceManager& GetVideoInterface() const; - VideoCommon::CustomAssetLoader& GetCustomAssetLoader() const; + VideoCommon::CustomResourceManager& GetCustomResourceManager() const; private: System(); diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index d7bb245d29..02a4894609 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -59,6 +59,7 @@ + @@ -669,12 +670,16 @@ + + + + @@ -814,6 +819,7 @@ + @@ -1320,14 +1326,15 @@ - + + diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.cpp b/Source/Core/VideoCommon/Assets/CustomAsset.cpp index 1591f93f91..7e2e817d15 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.cpp +++ b/Source/Core/VideoCommon/Assets/CustomAsset.cpp @@ -3,46 +3,54 @@ #include "VideoCommon/Assets/CustomAsset.h" +#include + namespace VideoCommon { CustomAsset::CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id) - : m_owning_library(std::move(library)), m_asset_id(asset_id) + const CustomAssetLibrary::AssetID& asset_id, u64 asset_handle) + : m_owning_library(std::move(library)), m_asset_id(asset_id), m_handle(asset_handle) { } -bool CustomAsset::Load() -{ - const auto load_information = LoadImpl(m_asset_id); - if (load_information.m_bytes_loaded > 0) - { - std::lock_guard lk(m_info_lock); - m_bytes_loaded = load_information.m_bytes_loaded; - m_last_loaded_time = load_information.m_load_time; - } - return load_information.m_bytes_loaded != 0; -} - -CustomAssetLibrary::TimeType CustomAsset::GetLastWriteTime() const -{ - return m_owning_library->GetLastAssetWriteTime(m_asset_id); -} - -const CustomAssetLibrary::TimeType& CustomAsset::GetLastLoadedTime() const +std::size_t CustomAsset::Load() { std::lock_guard lk(m_info_lock); + // The load time needs to come from before the data is actually read. + // Using a time point from after the read marks the asset as more up-to-date than it actually is, + // and has potential to race (and not be updated) if a change happens immediately after load. + const auto load_time = ClockType::now(); + + const auto load_information = LoadImpl(m_asset_id); + if (load_information.bytes_loaded > 0) + { + m_bytes_loaded = load_information.bytes_loaded; + m_last_loaded_time = load_time; + return m_bytes_loaded; + } + return 0; +} + +std::size_t CustomAsset::Unload() +{ + std::lock_guard lk(m_info_lock); + UnloadImpl(); + return std::exchange(m_bytes_loaded, 0); +} + +CustomAsset::TimeType CustomAsset::GetLastLoadedTime() const +{ return m_last_loaded_time; } +std::size_t CustomAsset::GetHandle() const +{ + return m_handle; +} + const CustomAssetLibrary::AssetID& CustomAsset::GetAssetId() const { return m_asset_id; } -std::size_t CustomAsset::GetByteSizeInMemory() const -{ - std::lock_guard lk(m_info_lock); - return m_bytes_loaded; -} - } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.h b/Source/Core/VideoCommon/Assets/CustomAsset.h index ea4a182932..cdf5f14e70 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.h +++ b/Source/Core/VideoCommon/Assets/CustomAsset.h @@ -6,9 +6,9 @@ #include "Common/CommonTypes.h" #include "VideoCommon/Assets/CustomAssetLibrary.h" +#include #include #include -#include namespace VideoCommon { @@ -17,42 +17,47 @@ namespace VideoCommon class CustomAsset { public: + using ClockType = std::chrono::steady_clock; + using TimeType = ClockType::time_point; + CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id); + const CustomAssetLibrary::AssetID& asset_id, u64 session_id); virtual ~CustomAsset() = default; CustomAsset(const CustomAsset&) = delete; CustomAsset(CustomAsset&&) = delete; CustomAsset& operator=(const CustomAsset&) = delete; CustomAsset& operator=(CustomAsset&&) = delete; - // Loads the asset from the library returning a pass/fail result - bool Load(); + // Loads the asset from the library returning the number of bytes loaded + std::size_t Load(); - // Queries the last time the asset was modified or standard epoch time - // if the asset hasn't been modified yet - // Note: not thread safe, expected to be called by the loader - CustomAssetLibrary::TimeType GetLastWriteTime() const; + // Unloads the asset data, resets the bytes loaded and + // returns the number of bytes unloaded + std::size_t Unload(); // Returns the time that the data was last loaded - const CustomAssetLibrary::TimeType& GetLastLoadedTime() const; + TimeType GetLastLoadedTime() const; // Returns an id that uniquely identifies this asset const CustomAssetLibrary::AssetID& GetAssetId() const; - // A rough estimate of how much space this asset - // will take in memroy - std::size_t GetByteSizeInMemory() const; + // Returns an id that is unique to this game session + // This is a faster form to hash and can be used + // as an index + std::size_t GetHandle() const; protected: const std::shared_ptr m_owning_library; private: virtual CustomAssetLibrary::LoadInfo LoadImpl(const CustomAssetLibrary::AssetID& asset_id) = 0; + virtual void UnloadImpl() = 0; CustomAssetLibrary::AssetID m_asset_id; + std::size_t m_handle; mutable std::mutex m_info_lock; std::size_t m_bytes_loaded = 0; - CustomAssetLibrary::TimeType m_last_loaded_time = {}; + std::atomic m_last_loaded_time = {}; }; // An abstract class that is expected to @@ -83,6 +88,14 @@ protected: bool m_loaded = false; mutable std::mutex m_data_lock; std::shared_ptr m_data; + +private: + void UnloadImpl() override + { + std::lock_guard lk(m_data_lock); + m_loaded = false; + m_data.reset(); + } }; // A helper struct that contains @@ -96,7 +109,7 @@ template struct CachedAsset { std::shared_ptr m_asset; - VideoCommon::CustomAssetLibrary::TimeType m_cached_write_time; + CustomAsset::TimeType m_cached_write_time; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/CustomAssetLibrary.cpp deleted file mode 100644 index 887875d2cb..0000000000 --- a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2023 Dolphin Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#include "VideoCommon/Assets/CustomAssetLibrary.h" - -#include - -#include "Common/Logging/Log.h" -#include "VideoCommon/Assets/TextureAsset.h" - -namespace VideoCommon -{ -CustomAssetLibrary::LoadInfo CustomAssetLibrary::LoadGameTexture(const AssetID& asset_id, - TextureData* data) -{ - const auto load_info = LoadTexture(asset_id, data); - if (load_info.m_bytes_loaded == 0) - return {}; - - if (data->m_type != TextureData::Type::Type_Texture2D) - { - ERROR_LOG_FMT( - VIDEO, - "Custom asset '{}' is not a valid game texture, it is expected to be a 2d texture " - "but was a '{}'.", - asset_id, data->m_type); - return {}; - } - - // Note: 'LoadTexture()' ensures we have a level loaded - for (std::size_t slice_index = 0; slice_index < data->m_texture.m_slices.size(); slice_index++) - { - auto& slice = data->m_texture.m_slices[slice_index]; - const auto& first_mip = slice.m_levels[0]; - - // Verify that each mip level is the correct size (divide by 2 each time). - u32 current_mip_width = first_mip.width; - u32 current_mip_height = first_mip.height; - for (u32 mip_level = 1; mip_level < static_cast(slice.m_levels.size()); mip_level++) - { - if (current_mip_width != 1 || current_mip_height != 1) - { - current_mip_width = std::max(current_mip_width / 2, 1u); - current_mip_height = std::max(current_mip_height / 2, 1u); - - const VideoCommon::CustomTextureData::ArraySlice::Level& level = slice.m_levels[mip_level]; - if (current_mip_width == level.width && current_mip_height == level.height) - continue; - - ERROR_LOG_FMT(VIDEO, - "Invalid custom game texture size {}x{} for texture asset {}. Slice {} with " - "mipmap level {} " - "must be {}x{}.", - level.width, level.height, asset_id, slice_index, mip_level, - current_mip_width, current_mip_height); - } - else - { - // It is invalid to have more than a single 1x1 mipmap. - ERROR_LOG_FMT( - VIDEO, - "Custom game texture {} has too many 1x1 mipmaps for slice {}. Skipping extra levels.", - asset_id, slice_index); - } - - // Drop this mip level and any others after it. - while (slice.m_levels.size() > mip_level) - slice.m_levels.pop_back(); - } - - // All levels have to have the same format. - if (std::ranges::any_of(slice.m_levels, - [&first_mip](const auto& l) { return l.format != first_mip.format; })) - { - ERROR_LOG_FMT( - VIDEO, "Custom game texture {} has inconsistent formats across mip levels for slice {}.", - asset_id, slice_index); - - return {}; - } - } - - return load_info; -} -} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h b/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h index b4831ea650..2ab8da408c 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h +++ b/Source/Core/VideoCommon/Assets/CustomAssetLibrary.h @@ -10,10 +10,11 @@ namespace VideoCommon { +class CustomTextureData; struct MaterialData; struct MeshData; struct PixelShaderData; -struct TextureData; +struct TextureAndSamplerData; // This class provides functionality to load // specific data (like textures). Where this data @@ -21,28 +22,21 @@ struct TextureData; class CustomAssetLibrary { public: - using TimeType = std::chrono::system_clock::time_point; - // The AssetID is a unique identifier for a particular asset using AssetID = std::string; struct LoadInfo { - std::size_t m_bytes_loaded = 0; - TimeType m_load_time = {}; + std::size_t bytes_loaded = 0; }; virtual ~CustomAssetLibrary() = default; + // Loads a texture with a sampler and type, if there are no levels, bytes loaded will be empty + virtual LoadInfo LoadTexture(const AssetID& asset_id, TextureAndSamplerData* data) = 0; + // Loads a texture, if there are no levels, bytes loaded will be empty - virtual LoadInfo LoadTexture(const AssetID& asset_id, TextureData* data) = 0; - - // Gets the last write time for a given asset id - virtual TimeType GetLastAssetWriteTime(const AssetID& asset_id) const = 0; - - // Loads a texture as a game texture, providing additional checks like confirming - // each mip level size is correct and that the format is consistent across the data - LoadInfo LoadGameTexture(const AssetID& asset_id, TextureData* data); + virtual LoadInfo LoadTexture(const AssetID& asset_id, CustomTextureData* data) = 0; // Loads a pixel shader virtual LoadInfo LoadPixelShader(const AssetID& asset_id, PixelShaderData* data) = 0; diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp index e70faa8359..2ca227546c 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp @@ -1,108 +1,157 @@ -// Copyright 2023 Dolphin Emulator Project +// Copyright 2025 Dolphin Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #include "VideoCommon/Assets/CustomAssetLoader.h" -#include "Common/MemoryUtil.h" -#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include + +#include "Common/Logging/Log.h" +#include "Common/Thread.h" + +#include "UICommon/UICommon.h" namespace VideoCommon { -void CustomAssetLoader::Init() +void CustomAssetLoader::Initialize() { - m_asset_monitor_thread_shutdown.Clear(); - - const size_t sys_mem = Common::MemPhysical(); - const size_t recommended_min_mem = 2 * size_t(1024 * 1024 * 1024); - // keep 2GB memory for system stability if system RAM is 4GB+ - use half of memory in other cases - m_max_memory_available = - (sys_mem / 2 < recommended_min_mem) ? (sys_mem / 2) : (sys_mem - recommended_min_mem); - - m_asset_monitor_thread = std::thread([this]() { - Common::SetCurrentThreadName("Asset monitor"); - while (true) - { - if (m_asset_monitor_thread_shutdown.IsSet()) - { - break; - } - - std::this_thread::sleep_for(TIME_BETWEEN_ASSET_MONITOR_CHECKS); - - std::lock_guard lk(m_asset_load_lock); - for (auto& [asset_id, asset_to_monitor] : m_assets_to_monitor) - { - if (auto ptr = asset_to_monitor.lock()) - { - const auto write_time = ptr->GetLastWriteTime(); - if (write_time > ptr->GetLastLoadedTime()) - { - (void)ptr->Load(); - } - } - } - } - }); - - m_asset_load_thread.Reset("Custom Asset Loader", [this](std::weak_ptr asset) { - if (auto ptr = asset.lock()) - { - if (m_memory_exceeded) - return; - - if (ptr->Load()) - { - std::lock_guard lk(m_asset_load_lock); - const std::size_t asset_memory_size = ptr->GetByteSizeInMemory(); - m_total_bytes_loaded += asset_memory_size; - m_assets_to_monitor.try_emplace(ptr->GetAssetId(), ptr); - if (m_total_bytes_loaded > m_max_memory_available) - { - ERROR_LOG_FMT(VIDEO, - "Asset memory exceeded with asset '{}', future assets won't load until " - "memory is available.", - ptr->GetAssetId()); - m_memory_exceeded = true; - } - } - } - }); + ResizeWorkerThreads(2); } void CustomAssetLoader::Shutdown() { - m_asset_load_thread.StopAndCancel(); - - m_asset_monitor_thread_shutdown.Set(); - m_asset_monitor_thread.join(); - m_assets_to_monitor.clear(); - m_total_bytes_loaded = 0; + Reset(false); } -std::shared_ptr -CustomAssetLoader::LoadGameTexture(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library) +bool CustomAssetLoader::StartWorkerThreads(u32 num_worker_threads) { - return LoadOrCreateAsset(asset_id, m_game_textures, std::move(library)); + for (u32 i = 0; i < num_worker_threads; i++) + { + m_worker_threads.emplace_back(&CustomAssetLoader::WorkerThreadRun, this, i); + } + + return HasWorkerThreads(); } -std::shared_ptr -CustomAssetLoader::LoadPixelShader(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library) +bool CustomAssetLoader::ResizeWorkerThreads(u32 num_worker_threads) { - return LoadOrCreateAsset(asset_id, m_pixel_shaders, std::move(library)); + if (m_worker_threads.size() == num_worker_threads) + return true; + + StopWorkerThreads(); + return StartWorkerThreads(num_worker_threads); } -std::shared_ptr -CustomAssetLoader::LoadMaterial(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library) +bool CustomAssetLoader::HasWorkerThreads() const { - return LoadOrCreateAsset(asset_id, m_materials, std::move(library)); + return !m_worker_threads.empty(); } -std::shared_ptr CustomAssetLoader::LoadMesh(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library) +void CustomAssetLoader::StopWorkerThreads() { - return LoadOrCreateAsset(asset_id, m_meshes, std::move(library)); + if (!HasWorkerThreads()) + return; + + // Signal worker threads to stop, and wake all of them. + { + std::lock_guard guard(m_assets_to_load_lock); + m_exit_flag.Set(); + m_worker_thread_wake.notify_all(); + } + + // Wait for worker threads to exit. + for (std::thread& thr : m_worker_threads) + thr.join(); + m_worker_threads.clear(); + m_exit_flag.Clear(); } + +void CustomAssetLoader::WorkerThreadRun(u32 thread_index) +{ + Common::SetCurrentThreadName(fmt::format("Asset Loader {}", thread_index).c_str()); + + std::unique_lock load_lock(m_assets_to_load_lock); + while (true) + { + m_worker_thread_wake.wait(load_lock, + [&] { return !m_assets_to_load.empty() || m_exit_flag.IsSet(); }); + + if (m_exit_flag.IsSet()) + return; + + // If more memory than allowed has already been loaded, we will load nothing more + // until the next ScheduleAssetsToLoad from Manager. + if (m_change_in_memory > m_allowed_memory) + { + m_assets_to_load.clear(); + continue; + } + + auto* const item = m_assets_to_load.front(); + m_assets_to_load.pop_front(); + + // Make sure another thread isn't loading this handle. + if (!m_handles_in_progress.insert(item->GetHandle()).second) + continue; + + load_lock.unlock(); + + // Unload previously loaded asset. + m_change_in_memory -= item->Unload(); + + const std::size_t bytes_loaded = item->Load(); + m_change_in_memory += s64(bytes_loaded); + + load_lock.lock(); + + { + INFO_LOG_FMT(VIDEO, "CustomAssetLoader thread {} loaded: {} ({})", thread_index, + item->GetAssetId(), UICommon::FormatSize(bytes_loaded)); + + std::lock_guard lk{m_assets_loaded_lock}; + m_asset_handles_loaded.emplace_back(item->GetHandle(), bytes_loaded > 0); + + // Make sure no other threads try to re-process this item. + // Manager will take the handles and re-ScheduleAssetsToLoad based on timestamps if needed. + std::erase(m_assets_to_load, item); + } + + m_handles_in_progress.erase(item->GetHandle()); + } +} + +auto CustomAssetLoader::TakeLoadResults() -> LoadResults +{ + std::lock_guard guard(m_assets_loaded_lock); + return {std::move(m_asset_handles_loaded), m_change_in_memory.exchange(0)}; +} + +void CustomAssetLoader::ScheduleAssetsToLoad(std::list assets_to_load, + u64 allowed_memory) +{ + if (assets_to_load.empty()) [[unlikely]] + return; + + // There's new assets to process, notify worker threads + std::lock_guard guard(m_assets_to_load_lock); + m_allowed_memory = allowed_memory; + m_assets_to_load = std::move(assets_to_load); + m_worker_thread_wake.notify_all(); +} + +void CustomAssetLoader::Reset(bool restart_worker_threads) +{ + const std::size_t worker_thread_count = m_worker_threads.size(); + StopWorkerThreads(); + + m_assets_to_load.clear(); + m_asset_handles_loaded.clear(); + m_allowed_memory = 0; + m_change_in_memory = 0; + + if (restart_worker_threads) + { + StartWorkerThreads(static_cast(worker_thread_count)); + } +} + } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h index 90d4f81a0e..8b67da65a9 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h @@ -1,27 +1,24 @@ -// Copyright 2023 Dolphin Emulator Project +// Copyright 2025 Dolphin Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once -#include -#include -#include +#include +#include +#include #include +#include #include +#include #include "Common/Flag.h" -#include "Common/Logging/Log.h" -#include "Common/WorkQueueThread.h" #include "VideoCommon/Assets/CustomAsset.h" -#include "VideoCommon/Assets/MaterialAsset.h" -#include "VideoCommon/Assets/MeshAsset.h" -#include "VideoCommon/Assets/ShaderAsset.h" -#include "VideoCommon/Assets/TextureAsset.h" namespace VideoCommon { -// This class is responsible for loading data asynchronously when requested -// and watches that data asynchronously reloading it if it changes +// This class takes any number of assets +// and loads them across a configurable +// thread pool class CustomAssetLoader { public: @@ -32,77 +29,54 @@ public: CustomAssetLoader& operator=(const CustomAssetLoader&) = delete; CustomAssetLoader& operator=(CustomAssetLoader&&) = delete; - void Init(); + void Initialize(); void Shutdown(); - // The following Load* functions will load or create an asset associated - // with the given asset id - // Loads happen asynchronously where the data will be set now or in the future - // Callees are expected to query the underlying data with 'GetData()' - // from the 'CustomLoadableAsset' class to determine if the data is ready for use - std::shared_ptr LoadGameTexture(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library); + using AssetHandle = std::pair; + struct LoadResults - std::shared_ptr LoadPixelShader(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library); + { + std::vector asset_handles; + s64 change_in_memory; + }; - std::shared_ptr LoadMaterial(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library); + // Returns a vector of loaded asset handle / loaded result pairs + // and the change in memory. + LoadResults TakeLoadResults(); - std::shared_ptr LoadMesh(const CustomAssetLibrary::AssetID& asset_id, - std::shared_ptr library); + // Schedule assets to load on the worker threads + // and set how much memory is available for loading these additional assets. + void ScheduleAssetsToLoad(std::list assets_to_load, u64 allowed_memory); + + void Reset(bool restart_worker_threads = true); private: - // TODO C++20: use a 'derived_from' concept against 'CustomAsset' when available - template - std::shared_ptr - LoadOrCreateAsset(const CustomAssetLibrary::AssetID& asset_id, - std::map>& asset_map, - std::shared_ptr library) - { - auto [it, inserted] = asset_map.try_emplace(asset_id); - if (!inserted) - { - auto shared = it->second.lock(); - if (shared) - return shared; - } - std::shared_ptr ptr(new AssetType(std::move(library), asset_id), [&](AssetType* a) { - { - std::lock_guard lk(m_asset_load_lock); - m_total_bytes_loaded -= a->GetByteSizeInMemory(); - m_assets_to_monitor.erase(a->GetAssetId()); - if (m_max_memory_available >= m_total_bytes_loaded && m_memory_exceeded) - { - INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading."); - m_memory_exceeded = false; - } - } - delete a; - }); - it->second = ptr; - m_asset_load_thread.Push(it->second); - return ptr; - } + bool StartWorkerThreads(u32 num_worker_threads); + bool ResizeWorkerThreads(u32 num_worker_threads); + bool HasWorkerThreads() const; + void StopWorkerThreads(); - static constexpr auto TIME_BETWEEN_ASSET_MONITOR_CHECKS = std::chrono::milliseconds{500}; + void WorkerThreadRun(u32 thread_index); - std::map> m_game_textures; - std::map> m_pixel_shaders; - std::map> m_materials; - std::map> m_meshes; - std::thread m_asset_monitor_thread; - Common::Flag m_asset_monitor_thread_shutdown; + Common::Flag m_exit_flag; - std::size_t m_total_bytes_loaded = 0; - std::size_t m_max_memory_available = 0; - std::atomic_bool m_memory_exceeded = false; + std::vector m_worker_threads; - std::map> m_assets_to_monitor; + std::mutex m_assets_to_load_lock; + std::list m_assets_to_load; - // Use a recursive mutex to handle the scenario where an asset goes out of scope while - // iterating over the assets to monitor which calls the lock above in 'LoadOrCreateAsset' - std::recursive_mutex m_asset_load_lock; - Common::WorkQueueThread> m_asset_load_thread; + std::condition_variable m_worker_thread_wake; + + std::vector m_asset_handles_loaded; + + // Memory available to load new assets. + s64 m_allowed_memory = 0; + + // Change in memory from just-loaded/unloaded asset results yet to be taken by the Manager. + std::atomic m_change_in_memory = 0; + + std::mutex m_assets_loaded_lock; + + std::set m_handles_in_progress; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp b/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp new file mode 100644 index 0000000000..f2af4ccbea --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomResourceManager.cpp @@ -0,0 +1,229 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "VideoCommon/Assets/CustomResourceManager.h" + +#include "Common/Logging/Log.h" +#include "Common/MemoryUtil.h" + +#include "UICommon/UICommon.h" + +#include "VideoCommon/Assets/CustomAsset.h" +#include "VideoCommon/Assets/TextureAsset.h" +#include "VideoCommon/VideoEvents.h" + +namespace VideoCommon +{ +void CustomResourceManager::Initialize() +{ + // Use half of available system memory but leave at least 2GiB unused for system stability. + constexpr size_t must_keep_unused = 2 * size_t(1024 * 1024 * 1024); + + const size_t sys_mem = Common::MemPhysical(); + const size_t keep_unused_mem = std::max(sys_mem / 2, std::min(sys_mem, must_keep_unused)); + + m_max_ram_available = sys_mem - keep_unused_mem; + + if (m_max_ram_available == 0) + ERROR_LOG_FMT(VIDEO, "Not enough system memory for custom resources."); + + m_asset_loader.Initialize(); + + m_xfb_event = + AfterFrameEvent::Register([this](Core::System&) { XFBTriggered(); }, "CustomResourceManager"); +} + +void CustomResourceManager::Shutdown() +{ + Reset(); + + m_asset_loader.Shutdown(); +} + +void CustomResourceManager::Reset() +{ + m_asset_loader.Reset(true); + + m_active_assets = {}; + m_pending_assets = {}; + m_asset_handle_to_data.clear(); + m_asset_id_to_handle.clear(); + m_texture_data_asset_cache.clear(); + m_dirty_assets.clear(); + m_ram_used = 0; +} + +void CustomResourceManager::MarkAssetDirty(const CustomAssetLibrary::AssetID& asset_id) +{ + std::lock_guard guard(m_dirty_mutex); + m_dirty_assets.insert(asset_id); +} + +CustomResourceManager::TextureTimePair CustomResourceManager::GetTextureDataFromAsset( + const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library) +{ + auto& resource = m_texture_data_asset_cache[asset_id]; + if (resource.asset_data != nullptr && + resource.asset_data->load_status == AssetData::LoadStatus::ResourceDataAvailable) + { + m_active_assets.MakeAssetHighestPriority(resource.asset->GetHandle(), resource.asset); + return {resource.texture_data, resource.asset->GetLastLoadedTime()}; + } + + // If there is an error, don't try and load again until the error is fixed + if (resource.asset_data != nullptr && resource.asset_data->has_load_error) + return {}; + + LoadTextureDataAsset(asset_id, std::move(library), &resource); + m_active_assets.MakeAssetHighestPriority(resource.asset->GetHandle(), resource.asset); + + return {}; +} + +void CustomResourceManager::LoadTextureDataAsset( + const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library, InternalTextureDataResource* resource) +{ + if (!resource->asset) + { + resource->asset = + CreateAsset(asset_id, AssetData::AssetType::TextureData, std::move(library)); + resource->asset_data = &m_asset_handle_to_data[resource->asset->GetHandle()]; + } + + auto texture_data = resource->asset->GetData(); + if (!texture_data || resource->asset_data->load_status == AssetData::LoadStatus::PendingReload) + { + // Tell the system we are still interested in loading this asset + const auto asset_handle = resource->asset->GetHandle(); + m_pending_assets.MakeAssetHighestPriority(asset_handle, + m_asset_handle_to_data[asset_handle].asset.get()); + } + else if (resource->asset_data->load_status == AssetData::LoadStatus::LoadFinished) + { + resource->texture_data = std::move(texture_data); + resource->asset_data->load_status = AssetData::LoadStatus::ResourceDataAvailable; + } +} + +void CustomResourceManager::XFBTriggered() +{ + ProcessDirtyAssets(); + ProcessLoadedAssets(); + + if (m_ram_used > m_max_ram_available) + { + RemoveAssetsUntilBelowMemoryLimit(); + } + + if (m_pending_assets.IsEmpty()) + return; + + if (m_ram_used > m_max_ram_available) + return; + + const u64 allowed_memory = m_max_ram_available - m_ram_used; + m_asset_loader.ScheduleAssetsToLoad(m_pending_assets.Elements(), allowed_memory); +} + +void CustomResourceManager::ProcessDirtyAssets() +{ + decltype(m_dirty_assets) dirty_assets; + + if (const auto lk = std::unique_lock{m_dirty_mutex, std::try_to_lock}) + std::swap(dirty_assets, m_dirty_assets); + + const auto now = CustomAsset::ClockType::now(); + for (const auto& asset_id : dirty_assets) + { + if (const auto it = m_asset_id_to_handle.find(asset_id); it != m_asset_id_to_handle.end()) + { + const auto asset_handle = it->second; + AssetData& asset_data = m_asset_handle_to_data[asset_handle]; + asset_data.load_status = AssetData::LoadStatus::PendingReload; + asset_data.load_request_time = now; + + // Asset was reloaded, clear any errors we might have + asset_data.has_load_error = false; + + m_pending_assets.InsertAsset(it->second, asset_data.asset.get()); + + DEBUG_LOG_FMT(VIDEO, "Dirty asset pending reload: {}", asset_data.asset->GetAssetId()); + } + } +} + +void CustomResourceManager::ProcessLoadedAssets() +{ + const auto load_results = m_asset_loader.TakeLoadResults(); + + // Update the ram with the change in memory from the loader + // + // Note: Assets with outstanding reload requests will have + // two copies in memory temporarily (the old data stored in + // the asset shared_ptr that the resource manager owns, and + // the new data loaded from the loader in the asset's shared_ptr) + // This temporary duplication will not be reflected in the + // resource manager's ram used + m_ram_used += load_results.change_in_memory; + + for (const auto& [handle, load_successful] : load_results.asset_handles) + { + AssetData& asset_data = m_asset_handle_to_data[handle]; + + // If we have a reload request that is newer than our loaded time + // we need to wait for another reload. + if (asset_data.load_request_time > asset_data.asset->GetLastLoadedTime()) + continue; + + m_pending_assets.RemoveAsset(handle); + + asset_data.load_request_time = {}; + if (!load_successful) + { + asset_data.has_load_error = true; + } + else + { + m_active_assets.InsertAsset(handle, asset_data.asset.get()); + asset_data.load_status = AssetData::LoadStatus::LoadFinished; + } + } +} + +void CustomResourceManager::RemoveAssetsUntilBelowMemoryLimit() +{ + const u64 threshold_ram = m_max_ram_available * 8 / 10; + + if (m_ram_used > threshold_ram) + { + INFO_LOG_FMT(VIDEO, "Memory usage over threshold: {}", UICommon::FormatSize(m_ram_used)); + } + + // Clear out least recently used resources until + // we get safely in our threshold + while (m_ram_used > threshold_ram && m_active_assets.Size() > 0) + { + auto* const asset = m_active_assets.RemoveLowestPriorityAsset(); + + AssetData& asset_data = m_asset_handle_to_data[asset->GetHandle()]; + + // Remove the resource manager's cached entry with its asset data + if (asset_data.type == AssetData::AssetType::TextureData) + { + m_texture_data_asset_cache.erase(asset->GetAssetId()); + } + // Remove the asset's copy + const std::size_t bytes_unloaded = asset_data.asset->Unload(); + m_ram_used -= bytes_unloaded; + + asset_data.load_status = AssetData::LoadStatus::Unloaded; + asset_data.load_request_time = {}; + + INFO_LOG_FMT(VIDEO, "Unloading asset: {} ({})", asset_data.asset->GetAssetId(), + UICommon::FormatSize(bytes_unloaded)); + } +} + +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomResourceManager.h b/Source/Core/VideoCommon/Assets/CustomResourceManager.h new file mode 100644 index 0000000000..6a2b5cbbe3 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/CustomResourceManager.h @@ -0,0 +1,215 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include +#include + +#include "Common/CommonTypes.h" +#include "Common/HookableEvent.h" + +#include "VideoCommon/Assets/CustomAsset.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/CustomAssetLoader.h" +#include "VideoCommon/Assets/CustomTextureData.h" + +namespace VideoCommon +{ +class TextureAsset; + +// The resource manager manages custom resources (textures, shaders, meshes) +// called assets. These assets are loaded using a priority system, +// where assets requested more often gets loaded first. This system +// also tracks memory usage and if memory usage goes over a calculated limit, +// then assets will be purged with older assets being targeted first. +class CustomResourceManager +{ +public: + void Initialize(); + void Shutdown(); + + void Reset(); + + // Request that an asset be reloaded + void MarkAssetDirty(const CustomAssetLibrary::AssetID& asset_id); + + void XFBTriggered(); + + using TextureTimePair = std::pair, CustomAsset::TimeType>; + + // Returns a pair with the custom texture data and the time it was last loaded + // Callees are not expected to hold onto the shared_ptr as that will prevent + // the resource manager from being able to properly release data + TextureTimePair GetTextureDataFromAsset(const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library); + +private: + // A generic interface to describe an assets' type + // and load state + struct AssetData + { + std::unique_ptr asset; + CustomAsset::TimeType load_request_time = {}; + bool has_load_error = false; + + enum class AssetType + { + TextureData + }; + AssetType type; + + enum class LoadStatus + { + PendingReload, + LoadFinished, + ResourceDataAvailable, + Unloaded, + }; + LoadStatus load_status = LoadStatus::PendingReload; + }; + + // A structure to represent some raw texture data + // (this data hasn't hit the GPU yet, used for custom textures) + struct InternalTextureDataResource + { + AssetData* asset_data = nullptr; + VideoCommon::TextureAsset* asset = nullptr; + std::shared_ptr texture_data; + }; + + void LoadTextureDataAsset(const CustomAssetLibrary::AssetID& asset_id, + std::shared_ptr library, + InternalTextureDataResource* resource); + + void ProcessDirtyAssets(); + void ProcessLoadedAssets(); + void RemoveAssetsUntilBelowMemoryLimit(); + + template + T* CreateAsset(const CustomAssetLibrary::AssetID& asset_id, AssetData::AssetType asset_type, + std::shared_ptr library) + { + const auto [it, added] = + m_asset_id_to_handle.try_emplace(asset_id, m_asset_handle_to_data.size()); + + if (added) + { + AssetData asset_data; + asset_data.asset = std::make_unique(library, asset_id, it->second); + asset_data.type = asset_type; + asset_data.load_request_time = {}; + asset_data.has_load_error = false; + + m_asset_handle_to_data.insert_or_assign(it->second, std::move(asset_data)); + } + auto& asset_data_from_handle = m_asset_handle_to_data[it->second]; + asset_data_from_handle.load_status = AssetData::LoadStatus::PendingReload; + + return static_cast(asset_data_from_handle.asset.get()); + } + + // Maintains a priority-sorted list of assets. + // Used to figure out which assets to load or unload first. + // Most recently used assets get marked with highest priority. + class AssetPriorityQueue + { + public: + const auto& Elements() const { return m_assets; } + + // Inserts or moves the asset to the top of the queue. + void MakeAssetHighestPriority(u64 asset_handle, CustomAsset* asset) + { + RemoveAsset(asset_handle); + m_assets.push_front(asset); + + // See CreateAsset for how a handle gets defined + if (asset_handle >= m_iterator_lookup.size()) + m_iterator_lookup.resize(asset_handle + 1, m_assets.end()); + + m_iterator_lookup[asset_handle] = m_assets.begin(); + } + + // Inserts an asset at lowest priority or + // does nothing if asset is already in the queue. + void InsertAsset(u64 asset_handle, CustomAsset* asset) + { + if (asset_handle >= m_iterator_lookup.size()) + m_iterator_lookup.resize(asset_handle + 1, m_assets.end()); + + if (m_iterator_lookup[asset_handle] == m_assets.end()) + { + m_assets.push_back(asset); + m_iterator_lookup[asset_handle] = std::prev(m_assets.end()); + } + } + + CustomAsset* RemoveLowestPriorityAsset() + { + if (m_assets.empty()) [[unlikely]] + return nullptr; + auto* const ret = m_assets.back(); + if (ret != nullptr) + { + m_iterator_lookup[ret->GetHandle()] = m_assets.end(); + } + m_assets.pop_back(); + return ret; + } + + void RemoveAsset(u64 asset_handle) + { + if (asset_handle >= m_iterator_lookup.size()) + return; + + const auto iter = m_iterator_lookup[asset_handle]; + if (iter != m_assets.end()) + { + m_assets.erase(iter); + m_iterator_lookup[asset_handle] = m_assets.end(); + } + } + + bool IsEmpty() const { return m_assets.empty(); } + + std::size_t Size() const { return m_assets.size(); } + + private: + std::list m_assets; + + // Handle-to-iterator lookup for fast access. + // Grows as needed on insert. + std::vector m_iterator_lookup; + }; + + // Assets that are currently active in memory, in order of most recently used by the game. + AssetPriorityQueue m_active_assets; + + // Assets that need to be loaded. + // e.g. Because the game tried to use them or because they changed on disk. + // Ordered by most recently used. + AssetPriorityQueue m_pending_assets; + + std::map m_asset_handle_to_data; + std::map m_asset_id_to_handle; + + // Memory used by currently "loaded" assets. + u64 m_ram_used = 0; + + // A calculated amount of memory to avoid exceeding. + u64 m_max_ram_available = 0; + + std::map m_texture_data_asset_cache; + + std::mutex m_dirty_mutex; + std::set m_dirty_assets; + + CustomAssetLoader m_asset_loader; + + Common::EventHook m_xfb_event; +}; + +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp index 90d77d8ec9..7933212d3a 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp @@ -13,30 +13,19 @@ #include "Common/JsonUtil.h" #include "Common/Logging/Log.h" #include "Common/StringUtil.h" +#include "Core/System.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/MaterialAsset.h" #include "VideoCommon/Assets/MeshAsset.h" #include "VideoCommon/Assets/ShaderAsset.h" #include "VideoCommon/Assets/TextureAsset.h" +#include "VideoCommon/Assets/TextureAssetUtils.h" #include "VideoCommon/RenderState.h" namespace VideoCommon { namespace { -std::chrono::system_clock::time_point FileTimeToSysTime(std::filesystem::file_time_type file_time) -{ -#ifdef _WIN32 - return std::chrono::clock_cast(file_time); -#else - // Note: all compilers should switch to chrono::clock_cast - // once it is available for use - const auto system_time_now = std::chrono::system_clock::now(); - const auto file_time_now = decltype(file_time)::clock::now(); - return std::chrono::time_point_cast( - file_time - file_time_now + system_time_now); -#endif -} - std::size_t GetAssetSize(const CustomTextureData& data) { std::size_t total = 0; @@ -50,30 +39,6 @@ std::size_t GetAssetSize(const CustomTextureData& data) return total; } } // namespace -CustomAssetLibrary::TimeType -DirectFilesystemAssetLibrary::GetLastAssetWriteTime(const AssetID& asset_id) const -{ - std::lock_guard lk(m_lock); - if (auto iter = m_assetid_to_asset_map_path.find(asset_id); - iter != m_assetid_to_asset_map_path.end()) - { - const auto& asset_map_path = iter->second; - CustomAssetLibrary::TimeType max_entry; - for (const auto& [key, value] : asset_map_path) - { - std::error_code ec; - const auto tp = std::filesystem::last_write_time(value, ec); - if (ec) - continue; - auto tp_sys = FileTimeToSysTime(tp); - if (tp_sys > max_entry) - max_entry = tp_sys; - } - return max_entry; - } - - return {}; -} CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadPixelShader(const AssetID& asset_id, PixelShaderData* data) @@ -158,7 +123,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadPixelShader(const if (!PixelShaderData::FromJson(asset_id, root_obj, data)) return {}; - return LoadInfo{approx_mem_size, GetLastAssetWriteTime(asset_id)}; + return LoadInfo{approx_mem_size}; } CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const AssetID& asset_id, @@ -216,7 +181,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const As return {}; } - return LoadInfo{metadata_size, GetLastAssetWriteTime(asset_id)}; + return LoadInfo{metadata_size}; } CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMesh(const AssetID& asset_id, @@ -311,11 +276,41 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMesh(const AssetI if (!MeshData::FromJson(asset_id, root_obj, data)) return {}; - return LoadInfo{approx_mem_size, GetLastAssetWriteTime(asset_id)}; + return LoadInfo{approx_mem_size}; } CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const AssetID& asset_id, - TextureData* data) + CustomTextureData* data) +{ + const auto asset_map = GetAssetMapForID(asset_id); + if (asset_map.empty()) + { + ERROR_LOG_FMT(VIDEO, "Asset '{}' error - raw texture expected to have one or two files mapped!", + asset_id); + return {}; + } + + const auto texture_path = asset_map.find("texture"); + + if (texture_path == asset_map.end()) + { + ERROR_LOG_FMT(VIDEO, "Asset '{}' expected to have a texture entry mapped!", asset_id); + return {}; + } + + if (!LoadTextureDataFromFile(asset_id, texture_path->second, + TextureAndSamplerData::Type::Type_Texture2D, data)) + { + return {}; + } + if (!PurgeInvalidMipsFromTextureData(asset_id, data)) + return {}; + + return LoadInfo{GetAssetSize(*data)}; +} + +CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const AssetID& asset_id, + TextureAndSamplerData* data) { const auto asset_map = GetAssetMapForID(asset_id); @@ -368,7 +363,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass } const auto& root_obj = root.get(); - if (!TextureData::FromJson(asset_id, root_obj, data)) + if (!TextureAndSamplerData::FromJson(asset_id, root_obj, data)) { return {}; } @@ -376,128 +371,62 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass else { data->m_sampler = RenderState::GetLinearSamplerState(); - data->m_type = TextureData::Type::Type_Texture2D; + data->m_type = TextureAndSamplerData::Type::Type_Texture2D; } - auto ext = PathToString(texture_path->second.extension()); - Common::ToLower(&ext); - if (ext == ".dds") - { - if (!LoadDDSTexture(&data->m_texture, PathToString(texture_path->second))) - { - ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load dds texture!", asset_id); - return {}; - } + if (!LoadTextureDataFromFile(asset_id, texture_path->second, data->m_type, &data->m_texture)) + return {}; + if (!PurgeInvalidMipsFromTextureData(asset_id, &data->m_texture)) + return {}; - if (data->m_texture.m_slices.empty()) [[unlikely]] - data->m_texture.m_slices.push_back({}); - - if (!LoadMips(texture_path->second, &data->m_texture.m_slices[0])) - return {}; - - return LoadInfo{GetAssetSize(data->m_texture) + metadata_size, GetLastAssetWriteTime(asset_id)}; - } - else if (ext == ".png") - { - // PNG could support more complicated texture types in the future - // but for now just error - if (data->m_type != TextureData::Type::Type_Texture2D) - { - ERROR_LOG_FMT(VIDEO, "Asset '{}' error - PNG is not supported for texture type '{}'!", - asset_id, data->m_type); - return {}; - } - - // If we have no slices, create one - if (data->m_texture.m_slices.empty()) - data->m_texture.m_slices.push_back({}); - - auto& slice = data->m_texture.m_slices[0]; - // If we have no levels, create one to pass into LoadPNGTexture - if (slice.m_levels.empty()) - slice.m_levels.push_back({}); - - if (!LoadPNGTexture(&slice.m_levels[0], PathToString(texture_path->second))) - { - ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load png texture!", asset_id); - return {}; - } - - if (!LoadMips(texture_path->second, &slice)) - return {}; - - return LoadInfo{GetAssetSize(data->m_texture) + metadata_size, GetLastAssetWriteTime(asset_id)}; - } - - ERROR_LOG_FMT(VIDEO, "Asset '{}' error - extension '{}' unknown!", asset_id, ext); - return {}; + return LoadInfo{GetAssetSize(data->m_texture) + metadata_size}; } void DirectFilesystemAssetLibrary::SetAssetIDMapData(const AssetID& asset_id, - AssetMap asset_path_map) + VideoCommon::Assets::AssetMap asset_path_map) { - std::lock_guard lk(m_lock); - m_assetid_to_asset_map_path[asset_id] = std::move(asset_path_map); -} - -bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_path, - CustomTextureData::ArraySlice* data) -{ - if (!data) [[unlikely]] - return false; - - std::string path; - std::string filename; - std::string extension; - SplitPath(PathToString(asset_path), &path, &filename, &extension); - - std::string extension_lower = extension; - Common::ToLower(&extension_lower); - - // Load additional mip levels - for (u32 mip_level = static_cast(data->m_levels.size());; mip_level++) + VideoCommon::Assets::AssetMap previous_asset_map; { - const auto mip_level_filename = filename + fmt::format("_mip{}", mip_level); - - const auto full_path = path + mip_level_filename + extension; - if (!File::Exists(full_path)) - return true; - - VideoCommon::CustomTextureData::ArraySlice::Level level; - if (extension_lower == ".dds") - { - if (!LoadDDSTexture(&level, full_path, mip_level)) - { - ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' failed to load", mip_level_filename); - return false; - } - } - else if (extension_lower == ".png") - { - if (!LoadPNGTexture(&level, full_path)) - { - ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' failed to load", mip_level_filename); - return false; - } - } - else - { - ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' has unsupported extension", mip_level_filename); - return false; - } - - data->m_levels.push_back(std::move(level)); + std::lock_guard lk(m_asset_map_lock); + previous_asset_map = m_asset_id_to_asset_map_path[asset_id]; } - return true; + { + std::lock_guard lk(m_path_map_lock); + for (const auto& [name, path] : previous_asset_map) + { + m_path_to_asset_id.erase(PathToString(path)); + } + + for (const auto& [name, path] : asset_path_map) + { + m_path_to_asset_id[PathToString(path)] = asset_id; + } + } + + { + std::lock_guard lk(m_asset_map_lock); + m_asset_id_to_asset_map_path[asset_id] = std::move(asset_path_map); + } } -DirectFilesystemAssetLibrary::AssetMap +void DirectFilesystemAssetLibrary::PathModified(std::string_view path) +{ + std::lock_guard lk(m_path_map_lock); + if (const auto iter = m_path_to_asset_id.find(path); iter != m_path_to_asset_id.end()) + { + auto& system = Core::System::GetInstance(); + auto& resource_manager = system.GetCustomResourceManager(); + resource_manager.MarkAssetDirty(iter->second); + } +} + +VideoCommon::Assets::AssetMap DirectFilesystemAssetLibrary::GetAssetMapForID(const AssetID& asset_id) const { - std::lock_guard lk(m_lock); - if (auto iter = m_assetid_to_asset_map_path.find(asset_id); - iter != m_assetid_to_asset_map_path.end()) + std::lock_guard lk(m_asset_map_lock); + if (auto iter = m_asset_id_to_asset_map_path.find(asset_id); + iter != m_asset_id_to_asset_map_path.end()) { return iter->second; } diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h index c4d99baf82..e3a8e81334 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h @@ -8,39 +8,39 @@ #include #include -#include "VideoCommon/Assets/CustomAssetLibrary.h" #include "VideoCommon/Assets/CustomTextureData.h" +#include "VideoCommon/Assets/TextureAsset.h" +#include "VideoCommon/Assets/Types.h" +#include "VideoCommon/Assets/WatchableFilesystemAssetLibrary.h" namespace VideoCommon { // This class implements 'CustomAssetLibrary' and loads any assets // directly from the filesystem -class DirectFilesystemAssetLibrary final : public CustomAssetLibrary +class DirectFilesystemAssetLibrary final : public WatchableFilesystemAssetLibrary { public: - using AssetMap = std::map; - - LoadInfo LoadTexture(const AssetID& asset_id, TextureData* data) override; + LoadInfo LoadTexture(const AssetID& asset_id, TextureAndSamplerData* data) override; + LoadInfo LoadTexture(const AssetID& asset_id, CustomTextureData* data) override; LoadInfo LoadPixelShader(const AssetID& asset_id, PixelShaderData* data) override; LoadInfo LoadMaterial(const AssetID& asset_id, MaterialData* data) override; LoadInfo LoadMesh(const AssetID& asset_id, MeshData* data) override; - // Gets the latest time from amongst all the files in the asset map - TimeType GetLastAssetWriteTime(const AssetID& asset_id) const override; - // Assigns the asset id to a map of files, how this map is read is dependent on the data // For instance, a raw texture would expect the map to have a single entry and load that // file as the asset. But a model file data might have its data spread across multiple files - void SetAssetIDMapData(const AssetID& asset_id, AssetMap asset_path_map); + void SetAssetIDMapData(const AssetID& asset_id, Assets::AssetMap asset_path_map); private: - // Loads additional mip levels into the texture structure until _mip texture is not found - bool LoadMips(const std::filesystem::path& asset_path, CustomTextureData::ArraySlice* data); + void PathModified(std::string_view path) override; // Gets the asset map given an asset id - AssetMap GetAssetMapForID(const AssetID& asset_id) const; + Assets::AssetMap GetAssetMapForID(const AssetID& asset_id) const; - mutable std::mutex m_lock; - std::map> m_assetid_to_asset_map_path; + mutable std::mutex m_asset_map_lock; + std::map m_asset_id_to_asset_map_path; + + mutable std::mutex m_path_map_lock; + std::map> m_path_to_asset_id; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/MaterialAsset.cpp b/Source/Core/VideoCommon/Assets/MaterialAsset.cpp index a9f2940deb..f69c884f35 100644 --- a/Source/Core/VideoCommon/Assets/MaterialAsset.cpp +++ b/Source/Core/VideoCommon/Assets/MaterialAsset.cpp @@ -384,7 +384,7 @@ CustomAssetLibrary::LoadInfo MaterialAsset::LoadImpl(const CustomAssetLibrary::A { auto potential_data = std::make_shared(); const auto loaded_info = m_owning_library->LoadMaterial(asset_id, potential_data.get()); - if (loaded_info.m_bytes_loaded == 0) + if (loaded_info.bytes_loaded == 0) return {}; { std::lock_guard lk(m_data_lock); diff --git a/Source/Core/VideoCommon/Assets/MeshAsset.cpp b/Source/Core/VideoCommon/Assets/MeshAsset.cpp index 05a4ba0961..bde8c8aab6 100644 --- a/Source/Core/VideoCommon/Assets/MeshAsset.cpp +++ b/Source/Core/VideoCommon/Assets/MeshAsset.cpp @@ -651,7 +651,7 @@ CustomAssetLibrary::LoadInfo MeshAsset::LoadImpl(const CustomAssetLibrary::Asset { auto potential_data = std::make_shared(); const auto loaded_info = m_owning_library->LoadMesh(asset_id, potential_data.get()); - if (loaded_info.m_bytes_loaded == 0) + if (loaded_info.bytes_loaded == 0) return {}; { std::lock_guard lk(m_data_lock); diff --git a/Source/Core/VideoCommon/Assets/ShaderAsset.cpp b/Source/Core/VideoCommon/Assets/ShaderAsset.cpp index dbe4c5cb02..50a4a8c95d 100644 --- a/Source/Core/VideoCommon/Assets/ShaderAsset.cpp +++ b/Source/Core/VideoCommon/Assets/ShaderAsset.cpp @@ -439,7 +439,7 @@ CustomAssetLibrary::LoadInfo PixelShaderAsset::LoadImpl(const CustomAssetLibrary { auto potential_data = std::make_shared(); const auto loaded_info = m_owning_library->LoadPixelShader(asset_id, potential_data.get()); - if (loaded_info.m_bytes_loaded == 0) + if (loaded_info.bytes_loaded == 0) return {}; { std::lock_guard lk(m_data_lock); diff --git a/Source/Core/VideoCommon/Assets/TextureAsset.cpp b/Source/Core/VideoCommon/Assets/TextureAsset.cpp index 335ed73b33..fc6c865f76 100644 --- a/Source/Core/VideoCommon/Assets/TextureAsset.cpp +++ b/Source/Core/VideoCommon/Assets/TextureAsset.cpp @@ -153,8 +153,8 @@ bool ParseSampler(const VideoCommon::CustomAssetLibrary::AssetID& asset_id, return true; } } // namespace -bool TextureData::FromJson(const CustomAssetLibrary::AssetID& asset_id, - const picojson::object& json, TextureData* data) +bool TextureAndSamplerData::FromJson(const CustomAssetLibrary::AssetID& asset_id, + const picojson::object& json, TextureAndSamplerData* data) { const auto type_iter = json.find("type"); if (type_iter == json.end()) @@ -176,7 +176,7 @@ bool TextureData::FromJson(const CustomAssetLibrary::AssetID& asset_id, if (type == "texture2d") { - data->m_type = TextureData::Type::Type_Texture2D; + data->m_type = TextureAndSamplerData::Type::Type_Texture2D; if (!ParseSampler(asset_id, json, &data->m_sampler)) { @@ -185,7 +185,7 @@ bool TextureData::FromJson(const CustomAssetLibrary::AssetID& asset_id, } else if (type == "texturecube") { - data->m_type = TextureData::Type::Type_TextureCube; + data->m_type = TextureAndSamplerData::Type::Type_TextureCube; } else { @@ -199,7 +199,7 @@ bool TextureData::FromJson(const CustomAssetLibrary::AssetID& asset_id, return true; } -void TextureData::ToJson(picojson::object* obj, const TextureData& data) +void TextureAndSamplerData::ToJson(picojson::object* obj, const TextureAndSamplerData& data) { if (!obj) [[unlikely]] return; @@ -207,13 +207,13 @@ void TextureData::ToJson(picojson::object* obj, const TextureData& data) auto& json_obj = *obj; switch (data.m_type) { - case TextureData::Type::Type_Texture2D: + case TextureAndSamplerData::Type::Type_Texture2D: json_obj.emplace("type", "texture2d"); break; - case TextureData::Type::Type_TextureCube: + case TextureAndSamplerData::Type::Type_TextureCube: json_obj.emplace("type", "texturecube"); break; - case TextureData::Type::Type_Undefined: + case TextureAndSamplerData::Type::Type_Undefined: break; }; @@ -254,11 +254,11 @@ void TextureData::ToJson(picojson::object* obj, const TextureData& data) json_obj.emplace("filter_mode", filter_mode); } -CustomAssetLibrary::LoadInfo GameTextureAsset::LoadImpl(const CustomAssetLibrary::AssetID& asset_id) +CustomAssetLibrary::LoadInfo TextureAsset::LoadImpl(const CustomAssetLibrary::AssetID& asset_id) { - auto potential_data = std::make_shared(); - const auto loaded_info = m_owning_library->LoadGameTexture(asset_id, potential_data.get()); - if (loaded_info.m_bytes_loaded == 0) + auto potential_data = std::make_shared(); + const auto loaded_info = m_owning_library->LoadTexture(asset_id, potential_data.get()); + if (loaded_info.bytes_loaded == 0) return {}; { std::lock_guard lk(m_data_lock); @@ -267,75 +267,4 @@ CustomAssetLibrary::LoadInfo GameTextureAsset::LoadImpl(const CustomAssetLibrary } return loaded_info; } - -bool GameTextureAsset::Validate(u32 native_width, u32 native_height) const -{ - std::lock_guard lk(m_data_lock); - - if (!m_loaded) - { - ERROR_LOG_FMT(VIDEO, - "Game texture can't be validated for asset '{}' because it is not loaded yet.", - GetAssetId()); - return false; - } - - if (m_data->m_texture.m_slices.empty()) - { - ERROR_LOG_FMT(VIDEO, - "Game texture can't be validated for asset '{}' because no data was available.", - GetAssetId()); - return false; - } - - if (m_data->m_texture.m_slices.size() > 1) - { - ERROR_LOG_FMT( - VIDEO, - "Game texture can't be validated for asset '{}' because it has more slices than expected.", - GetAssetId()); - return false; - } - - const auto& slice = m_data->m_texture.m_slices[0]; - if (slice.m_levels.empty()) - { - ERROR_LOG_FMT( - VIDEO, - "Game texture can't be validated for asset '{}' because first slice has no data available.", - GetAssetId()); - return false; - } - - // Verify that the aspect ratio of the texture hasn't changed, as this could have - // side-effects. - const VideoCommon::CustomTextureData::ArraySlice::Level& first_mip = slice.m_levels[0]; - if (first_mip.width * native_height != first_mip.height * native_width) - { - // Note: this feels like this should return an error but - // for legacy reasons this is only a notice that something *could* - // go wrong - WARN_LOG_FMT( - VIDEO, - "Invalid custom texture size {}x{} for game texture asset '{}'. The aspect differs " - "from the native size {}x{}.", - first_mip.width, first_mip.height, GetAssetId(), native_width, native_height); - } - - // Same deal if the custom texture isn't a multiple of the native size. - if (native_width != 0 && native_height != 0 && - (first_mip.width % native_width || first_mip.height % native_height)) - { - // Note: this feels like this should return an error but - // for legacy reasons this is only a notice that something *could* - // go wrong - WARN_LOG_FMT( - VIDEO, - "Invalid custom texture size {}x{} for game texture asset '{}'. Please use an integer " - "upscaling factor based on the native size {}x{}.", - first_mip.width, first_mip.height, GetAssetId(), native_width, native_height); - } - - return true; -} } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/TextureAsset.h b/Source/Core/VideoCommon/Assets/TextureAsset.h index e0929a79f0..f83d8ad109 100644 --- a/Source/Core/VideoCommon/Assets/TextureAsset.h +++ b/Source/Core/VideoCommon/Assets/TextureAsset.h @@ -13,11 +13,11 @@ namespace VideoCommon { -struct TextureData +struct TextureAndSamplerData { static bool FromJson(const CustomAssetLibrary::AssetID& asset_id, const picojson::object& json, - TextureData* data); - static void ToJson(picojson::object* obj, const TextureData& data); + TextureAndSamplerData* data); + static void ToJson(picojson::object* obj, const TextureAndSamplerData& data); enum class Type { Type_Undefined, @@ -30,23 +30,19 @@ struct TextureData SamplerState m_sampler; }; -class GameTextureAsset final : public CustomLoadableAsset +class TextureAsset final : public CustomLoadableAsset { public: using CustomLoadableAsset::CustomLoadableAsset; - // Validates that the game texture matches the native dimensions provided - // Callees are expected to call this once the data is loaded - bool Validate(u32 native_width, u32 native_height) const; - private: CustomAssetLibrary::LoadInfo LoadImpl(const CustomAssetLibrary::AssetID& asset_id) override; }; } // namespace VideoCommon template <> -struct fmt::formatter - : EnumFormatter +struct fmt::formatter + : EnumFormatter { constexpr formatter() : EnumFormatter({"Undefined", "Texture2D", "TextureCube"}) {} }; diff --git a/Source/Core/VideoCommon/Assets/TextureAssetUtils.cpp b/Source/Core/VideoCommon/Assets/TextureAssetUtils.cpp new file mode 100644 index 0000000000..5285c61bba --- /dev/null +++ b/Source/Core/VideoCommon/Assets/TextureAssetUtils.cpp @@ -0,0 +1,244 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "VideoCommon/Assets/TextureAssetUtils.h" + +#include + +#include "Common/FileUtil.h" +#include "Common/Logging/Log.h" +#include "Common/StringUtil.h" + +namespace VideoCommon +{ +namespace +{ +// Loads additional mip levels into the texture structure until _mip texture is not found +bool LoadMips(const std::filesystem::path& asset_path, CustomTextureData::ArraySlice* data) +{ + if (!data) [[unlikely]] + return false; + + std::string path; + std::string filename; + std::string extension; + SplitPath(PathToString(asset_path), &path, &filename, &extension); + + std::string extension_lower = extension; + Common::ToLower(&extension_lower); + + // Load additional mip levels + for (u32 mip_level = static_cast(data->m_levels.size());; mip_level++) + { + const auto mip_level_filename = filename + fmt::format("_mip{}", mip_level); + + const auto full_path = path + mip_level_filename + extension; + if (!File::Exists(full_path)) + return true; + + VideoCommon::CustomTextureData::ArraySlice::Level level; + if (extension_lower == ".dds") + { + if (!LoadDDSTexture(&level, full_path, mip_level)) + { + ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' failed to load", mip_level_filename); + return false; + } + } + else if (extension_lower == ".png") + { + if (!LoadPNGTexture(&level, full_path)) + { + ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' failed to load", mip_level_filename); + return false; + } + } + else + { + ERROR_LOG_FMT(VIDEO, "Custom mipmap '{}' has unsupported extension", mip_level_filename); + return false; + } + + data->m_levels.push_back(std::move(level)); + } + + return true; +} +} // namespace +bool LoadTextureDataFromFile(const CustomAssetLibrary::AssetID& asset_id, + const std::filesystem::path& asset_path, + TextureAndSamplerData::Type type, CustomTextureData* data) +{ + auto ext = PathToString(asset_path.extension()); + Common::ToLower(&ext); + if (ext == ".dds") + { + if (!LoadDDSTexture(data, PathToString(asset_path))) + { + ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load dds texture!", asset_id); + return false; + } + + if (data->m_slices.empty()) [[unlikely]] + data->m_slices.emplace_back(); + + if (!LoadMips(asset_path, data->m_slices.data())) + return false; + + return true; + } + + if (ext == ".png") + { + // PNG could support more complicated texture types in the future + // but for now just error + if (type != TextureAndSamplerData::Type::Type_Texture2D) + { + ERROR_LOG_FMT(VIDEO, "Asset '{}' error - PNG is not supported for texture type '{}'!", + asset_id, type); + return {}; + } + + // If we have no slices, create one + if (data->m_slices.empty()) + data->m_slices.emplace_back(); + + auto& slice = data->m_slices[0]; + // If we have no levels, create one to pass into LoadPNGTexture + if (slice.m_levels.empty()) + slice.m_levels.emplace_back(); + + if (!LoadPNGTexture(slice.m_levels.data(), PathToString(asset_path))) + { + ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load png texture!", asset_id); + return false; + } + + if (!LoadMips(asset_path, &slice)) + return false; + + return true; + } + + ERROR_LOG_FMT(VIDEO, "Asset '{}' error - extension '{}' unknown!", asset_id, ext); + return false; +} + +bool ValidateTextureData(const CustomAssetLibrary::AssetID& asset_id, const CustomTextureData& data, + u32 native_width, u32 native_height) +{ + if (data.m_slices.empty()) + { + ERROR_LOG_FMT(VIDEO, + "Texture data can't be validated for asset '{}' because no data was available.", + asset_id); + return false; + } + + if (data.m_slices.size() > 1) + { + ERROR_LOG_FMT( + VIDEO, + "Texture data can't be validated for asset '{}' because it has more slices than expected.", + asset_id); + return false; + } + + const auto& slice = data.m_slices[0]; + if (slice.m_levels.empty()) + { + ERROR_LOG_FMT( + VIDEO, + "Texture data can't be validated for asset '{}' because first slice has no data available.", + asset_id); + return false; + } + + // Verify that the aspect ratio of the texture hasn't changed, as this could have + // side-effects. + const CustomTextureData::ArraySlice::Level& first_mip = slice.m_levels[0]; + if (first_mip.width * native_height != first_mip.height * native_width) + { + // Note: this feels like this should return an error but + // for legacy reasons this is only a notice that something *could* + // go wrong + WARN_LOG_FMT(VIDEO, + "Invalid texture data size {}x{} for asset '{}'. The aspect differs " + "from the native size {}x{}.", + first_mip.width, first_mip.height, asset_id, native_width, native_height); + } + + // Same deal if the custom texture isn't a multiple of the native size. + if (native_width != 0 && native_height != 0 && + (first_mip.width % native_width || first_mip.height % native_height)) + { + // Note: this feels like this should return an error but + // for legacy reasons this is only a notice that something *could* + // go wrong + WARN_LOG_FMT(VIDEO, + "Invalid texture data size {}x{} for asset '{}'. Please use an integer " + "upscaling factor based on the native size {}x{}.", + first_mip.width, first_mip.height, asset_id, native_width, native_height); + } + + return true; +} + +bool PurgeInvalidMipsFromTextureData(const CustomAssetLibrary::AssetID& asset_id, + CustomTextureData* data) +{ + for (std::size_t slice_index = 0; slice_index < data->m_slices.size(); slice_index++) + { + auto& slice = data->m_slices[slice_index]; + const auto& first_mip = slice.m_levels[0]; + + // Verify that each mip level is the correct size (divide by 2 each time). + u32 current_mip_width = first_mip.width; + u32 current_mip_height = first_mip.height; + for (u32 mip_level = 1; mip_level < static_cast(slice.m_levels.size()); mip_level++) + { + if (current_mip_width != 1 || current_mip_height != 1) + { + current_mip_width = std::max(current_mip_width / 2, 1u); + current_mip_height = std::max(current_mip_height / 2, 1u); + + const VideoCommon::CustomTextureData::ArraySlice::Level& level = slice.m_levels[mip_level]; + if (current_mip_width == level.width && current_mip_height == level.height) + continue; + + ERROR_LOG_FMT(VIDEO, + "Invalid custom game texture size {}x{} for texture asset {}. Slice {} with " + "mipmap level {} " + "must be {}x{}.", + level.width, level.height, asset_id, slice_index, mip_level, + current_mip_width, current_mip_height); + } + else + { + // It is invalid to have more than a single 1x1 mipmap. + ERROR_LOG_FMT(VIDEO, + "Custom game texture {} has too many 1x1 mipmaps for slice {}. Skipping " + "extra levels.", + asset_id, slice_index); + } + + // Drop this mip level and any others after it. + while (slice.m_levels.size() > mip_level) + slice.m_levels.pop_back(); + } + + // All levels have to have the same format. + if (std::ranges::any_of(slice.m_levels, + [&first_mip](const auto& l) { return l.format != first_mip.format; })) + { + ERROR_LOG_FMT( + VIDEO, "Custom game texture {} has inconsistent formats across mip levels for slice {}.", + asset_id, slice_index); + + return false; + } + } + + return true; +} +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/TextureAssetUtils.h b/Source/Core/VideoCommon/Assets/TextureAssetUtils.h new file mode 100644 index 0000000000..ef7bc00751 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/TextureAssetUtils.h @@ -0,0 +1,22 @@ +// Copyright 2025 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include + +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/TextureAsset.h" + +namespace VideoCommon +{ +bool LoadTextureDataFromFile(const CustomAssetLibrary::AssetID& asset_id, + const std::filesystem::path& asset_path, + TextureAndSamplerData::Type type, CustomTextureData* data); + +bool ValidateTextureData(const CustomAssetLibrary::AssetID& asset_id, const CustomTextureData& data, + u32 native_width, u32 native_height); + +bool PurgeInvalidMipsFromTextureData(const CustomAssetLibrary::AssetID& asset_id, + CustomTextureData* data); +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/Types.h b/Source/Core/VideoCommon/Assets/Types.h new file mode 100644 index 0000000000..c10d45587c --- /dev/null +++ b/Source/Core/VideoCommon/Assets/Types.h @@ -0,0 +1,13 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include + +namespace VideoCommon::Assets +{ +using AssetMap = std::map; +} diff --git a/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h new file mode 100644 index 0000000000..196d311397 --- /dev/null +++ b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h @@ -0,0 +1,14 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "Common/FilesystemWatcher.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" + +namespace VideoCommon +{ +class WatchableFilesystemAssetLibrary : public CustomAssetLibrary, public Common::FilesystemWatcher +{ +}; +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/CMakeLists.txt b/Source/Core/VideoCommon/CMakeLists.txt index 6361e639c4..b4ba6ddf78 100644 --- a/Source/Core/VideoCommon/CMakeLists.txt +++ b/Source/Core/VideoCommon/CMakeLists.txt @@ -10,10 +10,11 @@ add_library(videocommon AbstractTexture.h Assets/CustomAsset.cpp Assets/CustomAsset.h - Assets/CustomAssetLibrary.cpp Assets/CustomAssetLibrary.h Assets/CustomAssetLoader.cpp Assets/CustomAssetLoader.h + Assets/CustomResourceManager.cpp + Assets/CustomResourceManager.h Assets/CustomTextureData.cpp Assets/CustomTextureData.h Assets/DirectFilesystemAssetLibrary.cpp @@ -26,6 +27,10 @@ add_library(videocommon Assets/ShaderAsset.h Assets/TextureAsset.cpp Assets/TextureAsset.h + Assets/TextureAssetUtils.cpp + Assets/TextureAssetUtils.h + Assets/Types.h + Assets/WatchableFilesystemAssetLibrary.h AsyncRequests.cpp AsyncRequests.h AsyncShaderCompiler.cpp diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h index 53d8d71892..d623a00e2e 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h +++ b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h @@ -7,12 +7,13 @@ #include -#include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/Types.h" struct GraphicsModAssetConfig { VideoCommon::CustomAssetLibrary::AssetID m_asset_id; - VideoCommon::DirectFilesystemAssetLibrary::AssetMap m_map; + VideoCommon::Assets::AssetMap m_map; void SerializeToConfig(picojson::object& json_obj) const; bool DeserializeFromConfig(const picojson::object& obj); diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp index 92c048860f..7366ad7ff7 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp +++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp @@ -16,7 +16,6 @@ #include "Core/System.h" #include "VideoCommon/AbstractGfx.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" #include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" #include "VideoCommon/ShaderGenCommon.h" #include "VideoCommon/TextureCacheBase.h" @@ -98,28 +97,6 @@ CustomPipelineAction::CustomPipelineAction( m_pipeline_passes.resize(m_passes_config.size()); } -void CustomPipelineAction::OnDrawStarted(GraphicsModActionData::DrawStarted* draw_started) +void CustomPipelineAction::OnDrawStarted(GraphicsModActionData::DrawStarted*) { - if (!draw_started) [[unlikely]] - return; - - if (!draw_started->custom_pixel_shader) [[unlikely]] - return; - - if (m_pipeline_passes.empty()) [[unlikely]] - return; - - auto& loader = Core::System::GetInstance().GetCustomAssetLoader(); - - // For now assume a single pass - const auto& pass_config = m_passes_config[0]; - auto& pass = m_pipeline_passes[0]; - - pass.UpdatePixelData(loader, m_library, draw_started->texture_units, - pass_config.m_pixel_material_asset); - CustomPixelShader custom_pixel_shader; - custom_pixel_shader.custom_shader = pass.m_last_generated_shader_code.GetBuffer(); - custom_pixel_shader.material_uniform_block = pass.m_last_generated_material_code.GetBuffer(); - *draw_started->custom_pixel_shader = custom_pixel_shader; - *draw_started->material_uniform_buffer = pass.m_material_data; } diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp index b52df66e31..f211442e96 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp +++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp @@ -12,7 +12,6 @@ #include "Common/VariantUtil.h" #include "VideoCommon/AbstractGfx.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" namespace { @@ -172,238 +171,8 @@ std::vector GlobalConflicts(std::string_view source) } // namespace -void CustomPipeline::UpdatePixelData( - VideoCommon::CustomAssetLoader& loader, - std::shared_ptr library, std::span texture_units, - const VideoCommon::CustomAssetLibrary::AssetID& material_to_load) +void CustomPipeline::UpdatePixelData(std::shared_ptr, + std::span, + const VideoCommon::CustomAssetLibrary::AssetID&) { - if (!m_pixel_material.m_asset || material_to_load != m_pixel_material.m_asset->GetAssetId()) - { - m_pixel_material.m_asset = loader.LoadMaterial(material_to_load, library); - } - - const auto material_data = m_pixel_material.m_asset->GetData(); - if (!material_data) - { - return; - } - - std::size_t max_material_data_size = 0; - if (m_pixel_material.m_asset->GetLastLoadedTime() > m_pixel_material.m_cached_write_time) - { - m_last_generated_material_code = ShaderCode{}; - m_pixel_material.m_cached_write_time = m_pixel_material.m_asset->GetLastLoadedTime(); - std::size_t texture_count = 0; - for (const auto& property : material_data->properties) - { - max_material_data_size += VideoCommon::MaterialProperty::GetMemorySize(property); - VideoCommon::MaterialProperty::WriteAsShaderCode(m_last_generated_material_code, property); - if (std::holds_alternative(property.m_value)) - { - texture_count++; - } - } - m_material_data.resize(max_material_data_size); - m_game_textures.resize(texture_count); - } - - if (!m_pixel_shader.m_asset || - m_pixel_shader.m_asset->GetLastLoadedTime() > m_pixel_shader.m_cached_write_time || - material_data->shader_asset != m_pixel_shader.m_asset->GetAssetId()) - { - m_pixel_shader.m_asset = loader.LoadPixelShader(material_data->shader_asset, library); - m_pixel_shader.m_cached_write_time = m_pixel_shader.m_asset->GetLastLoadedTime(); - - m_last_generated_shader_code = ShaderCode{}; - } - - const auto shader_data = m_pixel_shader.m_asset->GetData(); - if (!shader_data) - { - return; - } - - if (shader_data->m_properties.size() != material_data->properties.size()) - { - return; - } - - u8* material_buffer = m_material_data.data(); - u32 sampler_index = 8; - for (std::size_t index = 0; index < material_data->properties.size(); index++) - { - auto& property = material_data->properties[index]; - const auto shader_it = shader_data->m_properties.find(property.m_code_name); - if (shader_it == shader_data->m_properties.end()) - { - ERROR_LOG_FMT(VIDEO, - "Custom pipeline, has material asset '{}' that uses a " - "code name of '{}' but that can't be found on shader asset '{}'!", - m_pixel_material.m_asset->GetAssetId(), property.m_code_name, - m_pixel_shader.m_asset->GetAssetId()); - return; - } - - if (auto* texture_asset_id = - std::get_if(&property.m_value)) - { - if (*texture_asset_id != "") - { - auto asset = loader.LoadGameTexture(*texture_asset_id, library); - if (!asset) - { - return; - } - - auto& texture_asset = m_game_textures[index]; - if (!texture_asset || - texture_asset->m_cached_asset.m_asset->GetLastLoadedTime() > - texture_asset->m_cached_asset.m_cached_write_time || - *texture_asset_id != texture_asset->m_cached_asset.m_asset->GetAssetId()) - { - if (!texture_asset) - { - texture_asset = CachedTextureAsset{}; - } - const auto loaded_time = asset->GetLastLoadedTime(); - texture_asset->m_cached_asset = VideoCommon::CachedAsset{ - std::move(asset), loaded_time}; - texture_asset->m_texture.reset(); - - if (std::holds_alternative( - shader_it->second.m_default)) - { - texture_asset->m_sampler_code = - fmt::format("SAMPLER_BINDING({}) uniform sampler2D samp_{};\n", sampler_index, - property.m_code_name); - texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name); - } - else if (std::holds_alternative( - shader_it->second.m_default)) - { - texture_asset->m_sampler_code = - fmt::format("SAMPLER_BINDING({}) uniform sampler2DArray samp_{};\n", sampler_index, - property.m_code_name); - texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name); - } - else if (std::holds_alternative( - shader_it->second.m_default)) - { - texture_asset->m_sampler_code = - fmt::format("SAMPLER_BINDING({}) uniform samplerCube samp_{};\n", sampler_index, - property.m_code_name); - texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name); - } - } - - const auto texture_data = texture_asset->m_cached_asset.m_asset->GetData(); - if (!texture_data) - { - return; - } - - if (texture_asset->m_texture) - { - g_gfx->SetTexture(sampler_index, texture_asset->m_texture.get()); - g_gfx->SetSamplerState(sampler_index, texture_data->m_sampler); - } - else - { - AbstractTextureType texture_type = AbstractTextureType::Texture_2DArray; - if (std::holds_alternative( - shader_it->second.m_default)) - { - texture_type = AbstractTextureType::Texture_CubeMap; - } - else if (std::holds_alternative( - shader_it->second.m_default)) - { - texture_type = AbstractTextureType::Texture_2D; - } - - if (texture_data->m_texture.m_slices.empty() || - texture_data->m_texture.m_slices[0].m_levels.empty()) - { - return; - } - - auto& first_slice = texture_data->m_texture.m_slices[0]; - const TextureConfig texture_config( - first_slice.m_levels[0].width, first_slice.m_levels[0].height, - static_cast(first_slice.m_levels.size()), - static_cast(texture_data->m_texture.m_slices.size()), 1, - first_slice.m_levels[0].format, 0, texture_type); - texture_asset->m_texture = g_gfx->CreateTexture( - texture_config, fmt::format("Custom shader texture '{}'", property.m_code_name)); - if (texture_asset->m_texture) - { - for (std::size_t slice_index = 0; slice_index < texture_data->m_texture.m_slices.size(); - slice_index++) - { - auto& slice = texture_data->m_texture.m_slices[slice_index]; - for (u32 level_index = 0; level_index < static_cast(slice.m_levels.size()); - ++level_index) - { - auto& level = slice.m_levels[level_index]; - texture_asset->m_texture->Load(level_index, level.width, level.height, - level.row_length, level.data.data(), - level.data.size(), static_cast(slice_index)); - } - } - } - } - - sampler_index++; - } - } - else - { - VideoCommon::MaterialProperty::WriteToMemory(material_buffer, property); - } - } - - if (m_last_generated_shader_code.GetBuffer().empty()) - { - // Calculate shader details - std::string color_shader_data = - ReplaceAll(shader_data->m_shader_source, "custom_main", CUSTOM_PIXELSHADER_COLOR_FUNC); - const auto global_conflicts = GlobalConflicts(color_shader_data); - color_shader_data = ReplaceAll(color_shader_data, "\r\n", "\n"); - color_shader_data = ReplaceAll(color_shader_data, "{", "{{"); - color_shader_data = ReplaceAll(color_shader_data, "}", "}}"); - // First replace global conflicts with dummy strings - // This avoids the problem where a shorter word - // is in a longer word, ex two functions: 'execute' and 'execute_fast' - for (std::size_t i = 0; i < global_conflicts.size(); i++) - { - const std::string& identifier = global_conflicts[i]; - color_shader_data = - ReplaceAll(color_shader_data, identifier, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i)); - } - // Now replace the temporaries with the actual value - for (std::size_t i = 0; i < global_conflicts.size(); i++) - { - const std::string& identifier = global_conflicts[i]; - color_shader_data = ReplaceAll(color_shader_data, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i), - fmt::format("{}_{{0}}", identifier)); - } - - for (const auto& game_texture : m_game_textures) - { - if (!game_texture) - continue; - - m_last_generated_shader_code.Write("{}", game_texture->m_sampler_code); - m_last_generated_shader_code.Write("{}", game_texture->m_define_code); - } - - for (std::size_t i = 0; i < texture_units.size(); i++) - { - const auto& texture_unit = texture_units[i]; - m_last_generated_shader_code.Write( - "#define TEX_COORD{} data.texcoord[data.texmap_to_texcoord_index[{}]].xy\n", i, - texture_unit); - } - m_last_generated_shader_code.Write("{}", color_shader_data); - } } diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.h b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.h index 83bc9f84e1..3a904e0858 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.h +++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.h @@ -17,15 +17,9 @@ #include "VideoCommon/Assets/TextureAsset.h" #include "VideoCommon/ShaderGenCommon.h" -namespace VideoCommon -{ -class CustomAssetLoader; -} - struct CustomPipeline { - void UpdatePixelData(VideoCommon::CustomAssetLoader& loader, - std::shared_ptr library, + void UpdatePixelData(std::shared_ptr library, std::span texture_units, const VideoCommon::CustomAssetLibrary::AssetID& material_to_load); @@ -34,7 +28,7 @@ struct CustomPipeline struct CachedTextureAsset { - VideoCommon::CachedAsset m_cached_asset; + VideoCommon::CachedAsset m_cached_asset; std::unique_ptr m_texture; std::string m_sampler_code; std::string m_define_code; diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h index cecd0ce94f..f7c12fe6e5 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h +++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h @@ -47,7 +47,7 @@ struct TextureCreate std::string_view texture_name; u32 texture_width; u32 texture_height; - std::vector>* custom_textures; + std::vector>* custom_textures; // Dependencies needed to reload the texture and trigger this create again std::vector>* additional_dependencies; diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index b64c521e08..8acc3aff03 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -25,7 +25,6 @@ #include "Core/ConfigManager.h" #include "Core/System.h" #include "VideoCommon/Assets/CustomAsset.h" -#include "VideoCommon/Assets/CustomAssetLoader.h" #include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" #include "VideoCommon/OnScreenDisplay.h" #include "VideoCommon/VideoConfig.h" @@ -95,10 +94,11 @@ void HiresTexture::Update() GetTextureDirectoriesWithGameId(File::GetUserPath(D_HIRESTEXTURES_IDX), game_id); const std::vector extensions{".png", ".dds"}; - auto& system = Core::System::GetInstance(); - for (const auto& texture_directory : texture_directories) { + // Watch this directory for any texture reloads + s_file_library->Watch(texture_directory); + const auto texture_paths = Common::DoFileSearch({texture_directory}, extensions, /*recursive*/ true); @@ -130,10 +130,10 @@ void HiresTexture::Update() if (g_ActiveConfig.bCacheHiresTextures) { - auto hires_texture = std::make_shared( - has_arbitrary_mipmaps, - system.GetCustomAssetLoader().LoadGameTexture(filename, s_file_library)); - s_hires_texture_cache.try_emplace(filename, std::move(hires_texture)); + auto hires_texture = + std::make_shared(has_arbitrary_mipmaps, std::move(filename)); + static_cast(hires_texture->LoadTexture()); + s_hires_texture_cache.try_emplace(hires_texture->GetId(), hires_texture); } } } @@ -167,7 +167,7 @@ void HiresTexture::Clear() std::shared_ptr HiresTexture::Search(const TextureInfo& texture_info) { - const auto [base_filename, has_arb_mipmaps] = GetNameArbPair(texture_info); + auto [base_filename, has_arb_mipmaps] = GetNameArbPair(texture_info); if (base_filename == "") return nullptr; @@ -177,24 +177,27 @@ std::shared_ptr HiresTexture::Search(const TextureInfo& texture_in } else { - auto& system = Core::System::GetInstance(); - auto hires_texture = std::make_shared( - has_arb_mipmaps, - system.GetCustomAssetLoader().LoadGameTexture(base_filename, s_file_library)); + auto hires_texture = std::make_shared(has_arb_mipmaps, std::move(base_filename)); if (g_ActiveConfig.bCacheHiresTextures) { - s_hires_texture_cache.try_emplace(base_filename, hires_texture); + s_hires_texture_cache.try_emplace(hires_texture->GetId(), hires_texture); } return hires_texture; } } -HiresTexture::HiresTexture(bool has_arbitrary_mipmaps, - std::shared_ptr asset) - : m_has_arbitrary_mipmaps(has_arbitrary_mipmaps), m_game_texture(std::move(asset)) +HiresTexture::HiresTexture(bool has_arbitrary_mipmaps, std::string id) + : m_has_arbitrary_mipmaps(has_arbitrary_mipmaps), m_id(std::move(id)) { } +VideoCommon::CustomResourceManager::TextureTimePair HiresTexture::LoadTexture() const +{ + auto& system = Core::System::GetInstance(); + auto& custom_resource_manager = system.GetCustomResourceManager(); + return custom_resource_manager.GetTextureDataFromAsset(m_id, s_file_library); +} + std::set GetTextureDirectoriesWithGameId(const std::string& root_directory, const std::string& game_id) { diff --git a/Source/Core/VideoCommon/HiresTextures.h b/Source/Core/VideoCommon/HiresTextures.h index 5312285dd0..e99a3fc966 100644 --- a/Source/Core/VideoCommon/HiresTextures.h +++ b/Source/Core/VideoCommon/HiresTextures.h @@ -9,8 +9,8 @@ #include #include "Common/CommonTypes.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/CustomTextureData.h" -#include "VideoCommon/Assets/TextureAsset.h" #include "VideoCommon/TextureConfig.h" #include "VideoCommon/TextureInfo.h" @@ -27,12 +27,13 @@ public: static void Shutdown(); static std::shared_ptr Search(const TextureInfo& texture_info); - HiresTexture(bool has_arbitrary_mipmaps, std::shared_ptr asset); + HiresTexture(bool has_arbitrary_mipmaps, std::string id); bool HasArbitraryMipmaps() const { return m_has_arbitrary_mipmaps; } - const std::shared_ptr& GetAsset() const { return m_game_texture; } + VideoCommon::CustomResourceManager::TextureTimePair LoadTexture() const; + const std::string& GetId() const { return m_id; } private: bool m_has_arbitrary_mipmaps = false; - std::shared_ptr m_game_texture; + std::string m_id; }; diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp index b370481c04..f58460e9ad 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.cpp +++ b/Source/Core/VideoCommon/TextureCacheBase.cpp @@ -37,13 +37,14 @@ #include "VideoCommon/AbstractFramebuffer.h" #include "VideoCommon/AbstractGfx.h" #include "VideoCommon/AbstractStagingTexture.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/Assets/CustomTextureData.h" +#include "VideoCommon/Assets/TextureAssetUtils.h" #include "VideoCommon/BPMemory.h" #include "VideoCommon/FramebufferManager.h" #include "VideoCommon/GraphicsModSystem/Runtime/FBInfo.h" #include "VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h" #include "VideoCommon/GraphicsModSystem/Runtime/GraphicsModManager.h" -#include "VideoCommon/HiresTextures.h" #include "VideoCommon/OpcodeDecoding.h" #include "VideoCommon/PixelShaderManager.h" #include "VideoCommon/Present.h" @@ -262,25 +263,12 @@ void TextureCacheBase::SetBackupConfig(const VideoConfig& config) bool TextureCacheBase::DidLinkedAssetsChange(const TCacheEntry& entry) { - for (const auto& cached_asset : entry.linked_game_texture_assets) - { - if (cached_asset.m_asset) - { - if (cached_asset.m_asset->GetLastLoadedTime() > cached_asset.m_cached_write_time) - return true; - } - } + if (!entry.hires_texture) + return false; - for (const auto& cached_asset : entry.linked_asset_dependencies) - { - if (cached_asset.m_asset) - { - if (cached_asset.m_asset->GetLastLoadedTime() > cached_asset.m_cached_write_time) - return true; - } - } + const auto [texture_data, load_time] = entry.hires_texture->LoadTexture(); - return false; + return load_time > entry.last_load_time; } RcTcacheEntry TextureCacheBase::ApplyPaletteToEntry(RcTcacheEntry& entry, const u8* palette, @@ -1566,80 +1554,50 @@ RcTcacheEntry TextureCacheBase::GetTexture(const int textureCacheSafetyColorSamp InvalidateTexture(oldest_entry); } - std::vector> cached_game_assets; - std::vector> data_for_assets; + std::shared_ptr hires_texture; bool has_arbitrary_mipmaps = false; bool skip_texture_dump = false; - std::shared_ptr hires_texture; + std::shared_ptr custom_texture_data = nullptr; + VideoCommon::CustomAsset::TimeType load_time = {}; if (g_ActiveConfig.bHiresTextures) { hires_texture = HiresTexture::Search(texture_info); if (hires_texture) { - auto asset = hires_texture->GetAsset(); - const auto loaded_time = asset->GetLastLoadedTime(); - cached_game_assets.push_back( - VideoCommon::CachedAsset{std::move(asset), loaded_time}); has_arbitrary_mipmaps = hires_texture->HasArbitraryMipmaps(); + std::tie(custom_texture_data, load_time) = hires_texture->LoadTexture(); + if (custom_texture_data && !VideoCommon::ValidateTextureData( + hires_texture->GetId(), *custom_texture_data, + texture_info.GetRawWidth(), texture_info.GetRawHeight())) + { + custom_texture_data = nullptr; + load_time = {}; + } skip_texture_dump = true; } } - std::vector> additional_dependencies; - std::string texture_name = ""; if (g_ActiveConfig.bGraphicMods) { u32 height = texture_info.GetRawHeight(); u32 width = texture_info.GetRawWidth(); - if (hires_texture) - { - auto asset = hires_texture->GetAsset(); - if (asset) - { - auto data = asset->GetData(); - if (data) - { - if (!data->m_texture.m_slices.empty()) - { - if (!data->m_texture.m_slices[0].m_levels.empty()) - { - height = data->m_texture.m_slices[0].m_levels[0].height; - width = data->m_texture.m_slices[0].m_levels[0].width; - } - } - } - } - } texture_name = texture_info.CalculateTextureName().GetFullName(); - GraphicsModActionData::TextureCreate texture_create{ - texture_name, width, height, &cached_game_assets, &additional_dependencies}; + GraphicsModActionData::TextureCreate texture_create{texture_name, width, height, nullptr, + nullptr}; for (const auto& action : g_graphics_mod_manager->GetTextureCreateActions(texture_name)) { action->OnTextureCreate(&texture_create); } } - data_for_assets.reserve(cached_game_assets.size()); - for (auto& cached_asset : cached_game_assets) - { - auto data = cached_asset.m_asset->GetData(); - if (data) - { - if (cached_asset.m_asset->Validate(texture_info.GetRawWidth(), texture_info.GetRawHeight())) - { - data_for_assets.push_back(data); - } - } - } - auto entry = CreateTextureEntry(TextureCreationInfo{base_hash, full_hash, bytes_per_block, palette_size}, - texture_info, textureCacheSafetyColorSampleSize, - std::move(data_for_assets), has_arbitrary_mipmaps, skip_texture_dump); - entry->linked_game_texture_assets = std::move(cached_game_assets); - entry->linked_asset_dependencies = std::move(additional_dependencies); + texture_info, textureCacheSafetyColorSampleSize, custom_texture_data.get(), + has_arbitrary_mipmaps, skip_texture_dump); + entry->hires_texture = std::move(hires_texture); + entry->last_load_time = load_time; entry->texture_info_name = std::move(texture_name); return entry; } @@ -1649,8 +1607,7 @@ RcTcacheEntry TextureCacheBase::GetTexture(const int textureCacheSafetyColorSamp // expected because each texture is loaded into a texture array RcTcacheEntry TextureCacheBase::CreateTextureEntry( const TextureCreationInfo& creation_info, const TextureInfo& texture_info, - const int safety_color_sample_size, - std::vector> assets_data, + const int safety_color_sample_size, VideoCommon::CustomTextureData* custom_texture_data, const bool custom_arbitrary_mipmaps, bool skip_texture_dump) { #ifdef __APPLE__ @@ -1660,33 +1617,22 @@ RcTcacheEntry TextureCacheBase::CreateTextureEntry( #endif RcTcacheEntry entry; - if (!assets_data.empty()) + if (custom_texture_data) { - const auto calculate_max_levels = [&]() { - const auto max_element = std::ranges::max_element( - assets_data, {}, [](const auto& v) { return v->m_texture.m_slices[0].m_levels.size(); }); - return (*max_element)->m_texture.m_slices[0].m_levels.size(); - }; - const u32 texLevels = no_mips ? 1 : (u32)calculate_max_levels(); - const auto& first_level = assets_data[0]->m_texture.m_slices[0].m_levels[0]; - const TextureConfig config(first_level.width, first_level.height, texLevels, - static_cast(assets_data.size()), 1, first_level.format, 0, - AbstractTextureType::Texture_2DArray); + const u32 texLevels = no_mips ? 1 : (u32)custom_texture_data->m_slices[0].m_levels.size(); + const auto& first_level = custom_texture_data->m_slices[0].m_levels[0]; + const TextureConfig config(first_level.width, first_level.height, texLevels, 1, 1, + first_level.format, 0, AbstractTextureType::Texture_2DArray); entry = AllocateCacheEntry(config); if (!entry) [[unlikely]] return entry; - for (u32 data_index = 0; data_index < static_cast(assets_data.size()); data_index++) + const auto& slice = custom_texture_data->m_slices[0]; + for (u32 level_index = 0; + level_index < std::min(texLevels, static_cast(slice.m_levels.size())); ++level_index) { - const auto& asset = assets_data[data_index]; - const auto& slice = asset->m_texture.m_slices[0]; - for (u32 level_index = 0; - level_index < std::min(texLevels, static_cast(slice.m_levels.size())); - ++level_index) - { - const auto& level = slice.m_levels[level_index]; - entry->texture->Load(level_index, level.width, level.height, level.row_length, - level.data.data(), level.data.size(), data_index); - } + const auto& level = slice.m_levels[level_index]; + entry->texture->Load(level_index, level.width, level.height, level.row_length, + level.data.data(), level.data.size()); } entry->has_arbitrary_mips = custom_arbitrary_mipmaps; diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h index 532feaca7d..dd6cb46368 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.h +++ b/Source/Core/VideoCommon/TextureCacheBase.h @@ -24,6 +24,7 @@ #include "VideoCommon/AbstractTexture.h" #include "VideoCommon/Assets/CustomAsset.h" #include "VideoCommon/BPMemory.h" +#include "VideoCommon/HiresTextures.h" #include "VideoCommon/TextureConfig.h" #include "VideoCommon/TextureDecoder.h" #include "VideoCommon/TextureInfo.h" @@ -167,8 +168,8 @@ struct TCacheEntry std::string texture_info_name = ""; - std::vector> linked_game_texture_assets; - std::vector> linked_asset_dependencies; + VideoCommon::CustomAsset::TimeType last_load_time; + std::shared_ptr hires_texture; explicit TCacheEntry(std::unique_ptr tex, std::unique_ptr fb); @@ -351,11 +352,10 @@ private: void SetBackupConfig(const VideoConfig& config); - RcTcacheEntry - CreateTextureEntry(const TextureCreationInfo& creation_info, const TextureInfo& texture_info, - int safety_color_sample_size, - std::vector> assets_data, - bool custom_arbitrary_mipmaps, bool skip_texture_dump); + RcTcacheEntry CreateTextureEntry(const TextureCreationInfo& creation_info, + const TextureInfo& texture_info, int safety_color_sample_size, + VideoCommon::CustomTextureData* custom_texture_data, + bool custom_arbitrary_mipmaps, bool skip_texture_dump); RcTcacheEntry GetXFBFromCache(u32 address, u32 width, u32 height, u32 stride); diff --git a/Source/Core/VideoCommon/VideoBackendBase.cpp b/Source/Core/VideoCommon/VideoBackendBase.cpp index 73035e0e1b..a4cc35fa09 100644 --- a/Source/Core/VideoCommon/VideoBackendBase.cpp +++ b/Source/Core/VideoCommon/VideoBackendBase.cpp @@ -41,6 +41,7 @@ #endif #include "VideoCommon/AbstractGfx.h" +#include "VideoCommon/Assets/CustomResourceManager.h" #include "VideoCommon/AsyncRequests.h" #include "VideoCommon/BPStructs.h" #include "VideoCommon/BoundingBox.h" @@ -341,12 +342,16 @@ bool VideoBackendBase::InitializeShared(std::unique_ptr gfx, } g_shader_cache->InitializeShaderCache(); + system.GetCustomResourceManager().Initialize(); return true; } void VideoBackendBase::ShutdownShared() { + auto& system = Core::System::GetInstance(); + system.GetCustomResourceManager().Shutdown(); + g_frame_dumper.reset(); g_presenter.reset(); @@ -369,7 +374,6 @@ void VideoBackendBase::ShutdownShared() m_initialized = false; - auto& system = Core::System::GetInstance(); VertexLoaderManager::Clear(); system.GetFifo().Shutdown(); } diff --git a/Source/VSProps/Base.Dolphin.props b/Source/VSProps/Base.Dolphin.props index f2f213b009..ce65bf1911 100644 --- a/Source/VSProps/Base.Dolphin.props +++ b/Source/VSProps/Base.Dolphin.props @@ -17,6 +17,7 @@ $(ExternalsDir)rangeset\include;%(AdditionalIncludeDirectories) $(ExternalsDir)Vulkan-Headers\include;%(AdditionalIncludeDirectories) $(ExternalsDir)VulkanMemoryAllocator\include;%(AdditionalIncludeDirectories) + $(ExternalsDir)watcher\watcher\include;%(AdditionalIncludeDirectories) $(ExternalsDir)WIL\include;%(AdditionalIncludeDirectories) WIL_SUPPRESS_EXCEPTIONS;%(PreprocessorDefinitions)