replace DoFileSearch with optimized version

Source/Core/Common/FileSearch.cpp

@@ -7,10 +7,19 @@
 
 #include "Common/CommonPaths.h"
 #include "Common/FileSearch.h"
+
+#ifdef _MSC_VER
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#define HAS_STD_FILESYSTEM
+#else
 #include "Common/FileUtil.h"
+#endif
 
 namespace Common
 {
+#ifndef HAS_STD_FILESYSTEM
+
 static std::vector<std::string>
 FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
                    std::function<bool(const File::FSTEntry&)> callback)
@@ -36,10 +45,10 @@ FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
   return result;
 }
 
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
+std::vector<std::string> DoFileSearchNoSTL(const std::vector<std::string>& directories,
-                                      const std::vector<std::string>& directories, bool recursive)
+                                           const std::vector<std::string>& exts, bool recursive)
 {
-  bool accept_all = std::find(exts.begin(), exts.end(), "") != exts.end();
+  bool accept_all = exts.empty();
   return FileSearchWithTest(directories, recursive, [&](const File::FSTEntry& entry) {
     if (accept_all)
       return true;
@@ -52,11 +61,71 @@ std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
   });
 }
 
-// Result includes the passed directories themselves as well as their subdirectories.
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
-                                            bool recursive)
 {
-  return FileSearchWithTest(directories, true,
+  return DoFileSearchNoSTL(directories, exts, recursive);
-                            [&](const File::FSTEntry& entry) { return entry.isDirectory; });
 }
+
+#else
+
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
+{
+  bool accept_all = exts.empty();
+
+  std::vector<fs::path> native_exts;
+  for (const auto& ext : exts)
+    native_exts.push_back(ext);
+
+  // N.B. This avoids doing any copies
+  auto ext_matches = [&native_exts](const fs::path& path) {
+    const auto& native_path = path.native();
+    return std::any_of(native_exts.cbegin(), native_exts.cend(), [&native_path](const auto& ext) {
+      // TODO provide cross-platform compat for the comparison function, once more platforms
+      // support std::filesystem
+      return native_path.length() >= ext.native().length() &&
+             _wcsicmp(&native_path.c_str()[native_path.length() - ext.native().length()],
+                      ext.c_str()) == 0;
+    });
+  };
+
+  std::vector<std::string> result;
+  auto add_filtered = [&](const fs::directory_entry& entry) {
+    auto& path = entry.path();
+    if (accept_all || (ext_matches(path) && !fs::is_directory(path)))
+      result.emplace_back(path.u8string());
+  };
+  for (const auto& directory : directories)
+  {
+    if (recursive)
+    {
+      // TODO use fs::directory_options::follow_directory_symlink ?
+      for (auto& entry : fs::recursive_directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+    else
+    {
+      for (auto& entry : fs::directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+  }
+
+  // Remove duplicates (occurring because caller gave e.g. duplicate or overlapping directories -
+  // not because std::filesystem returns duplicates). Also note that this pathname-based uniqueness
+  // isn't as thorough as std::filesystem::equivalent.
+  std::sort(result.begin(), result.end());
+  result.erase(std::unique(result.begin(), result.end()), result.end());
+
+  // Dolphin expects to be able to use "/" (DIR_SEP) everywhere. std::filesystem uses the OS
+  // separator.
+  if (fs::path::preferred_separator != DIR_SEP_CHR)
+    for (auto& path : result)
+      std::replace(path.begin(), path.end(), '\\', DIR_SEP_CHR);
+
+  return result;
+}
+
+#endif
+
 }  // namespace Common

Source/Core/Common/FileSearch.h

@@ -9,9 +9,9 @@
 
 namespace Common
 {
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
+// Callers can pass empty "exts" to indicate they want all files + directories in results
-                                      const std::vector<std::string>& directories,
+// Otherwise, only files matching the extensions are returned
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts = {},
                                       bool recursive = false);
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
-                                            bool recursive);
 }  // namespace Common

Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp

@@ -148,7 +148,7 @@ GCMemcardDirectory::GCMemcardDirectory(const std::string& directory, int slot, u
     hdr_file.ReadBytes(&m_hdr, BLOCK_SIZE);
   }
 
-  std::vector<std::string> filenames = Common::DoFileSearch({".gci"}, {m_save_directory});
+  std::vector<std::string> filenames = Common::DoFileSearch({m_save_directory}, {".gci"});
 
   if (filenames.size() > 112)
   {

Source/Core/DolphinQt2/Settings.cpp

@@ -365,7 +365,7 @@ QVector<QString> Settings::GetProfiles(const InputConfig* config) const
   const std::string path = GetProfilesDir().toStdString() + config->GetProfileName();
   QVector<QString> vec;
 
-  for (const auto& file : Common::DoFileSearch({".ini"}, {path}))
+  for (const auto& file : Common::DoFileSearch({path}, {".ini"}))
   {
     std::string basename;
     SplitPath(file, nullptr, &basename, nullptr);

Source/Core/DolphinQt2/Settings/InterfacePane.cpp

@@ -59,9 +59,8 @@ void InterfacePane::CreateUI()
   combobox_layout->addRow(tr("&Theme:"), m_combobox_theme);
 
   // List avalable themes
-  auto file_search_results = Common::DoFileSearch(
+  auto file_search_results =
-      {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR},
+      Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR});
-      /*recursive*/ false);
   for (const std::string& filename : file_search_results)
   {
     std::string name, ext;

Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp

@@ -196,9 +196,8 @@ void InterfaceConfigPane::LoadGUIValues()
 
 void InterfaceConfigPane::LoadThemes()
 {
-  auto sv = Common::DoFileSearch(
+  auto sv =
-      {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR},
+      Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR});
-      /*recursive*/ false);
   for (const std::string& filename : sv)
   {
     std::string name, ext;

Source/Core/DolphinWX/FrameTools.cpp

@@ -1687,7 +1687,7 @@ void CFrame::GameListChanged(wxCommandEvent& event)
     break;
   case IDM_PURGE_GAME_LIST_CACHE:
     std::vector<std::string> filenames =
-        Common::DoFileSearch({".cache"}, {File::GetUserPath(D_CACHE_IDX)});
+        Common::DoFileSearch({File::GetUserPath(D_CACHE_IDX)}, {".cache"});
 
     for (const std::string& filename : filenames)
     {

Source/Core/DolphinWX/GameListCtrl.cpp

@@ -743,10 +743,9 @@ void GameListCtrl::RescanList()
   const std::vector<std::string> search_extensions = {".gcm",  ".tgc", ".iso", ".ciso", ".gcz",
                                                       ".wbfs", ".wad", ".dol", ".elf"};
   // TODO This could process paths iteratively as they are found
-  auto search_results = Common::DoFileSearch(search_extensions, SConfig::GetInstance().m_ISOFolder,
+  auto search_results = Common::DoFileSearch(SConfig::GetInstance().m_ISOFolder, search_extensions,
                                              SConfig::GetInstance().m_RecursiveISOFolder);
 
-  // TODO rethink some good algorithms to use here
   std::vector<std::string> cached_paths;
   for (const auto& file : m_cached_files)
     cached_paths.emplace_back(file->GetFileName());
@@ -761,14 +760,8 @@ void GameListCtrl::RescanList()
                       cached_paths.cend(), std::back_inserter(new_paths));
 
   const Core::TitleDatabase title_database;
-  // TODO we could store all paths and modification times to judge if file needs to be rescanned.
-  // If we cached paths that turned out to be invalid, this would save failing on them each time
-  // refresh is done.
-  // However if people e.g. set dolphin to recursively scan the root of their drive(s), then we
-  // would cache way too much data. Maybe just use an upper bound of invalid paths to cache?
   // For now, only scan new_paths. This could cause false negatives (file actively being written),
-  // but otherwise
+  // but otherwise should be fine.
-  // should be fine.
   for (const auto& path : removed_paths)
   {
     auto it = std::find_if(

Source/Core/DolphinWX/GameListCtrl.h

@@ -7,6 +7,7 @@
 #include <cstddef>
 #include <memory>
 #include <string>
+#include <thread>
 #include <vector>
 
 #include <wx/listctrl.h>
@@ -14,6 +15,7 @@
 
 #include "Common/ChunkFile.h"
 #include "Common/Event.h"
+#include "Common/Flag.h"
 #include "DolphinWX/ISOFile.h"
 
 class wxEmuStateTip : public wxTipWindow

Source/Core/DolphinWX/Input/InputConfigDiag.cpp

@@ -264,7 +264,7 @@ void InputConfigDialog::UpdateProfileComboBox()
   pname += PROFILES_PATH;
   pname += m_config.GetProfileName();
 
-  std::vector<std::string> sv = Common::DoFileSearch({".ini"}, {pname});
+  std::vector<std::string> sv = Common::DoFileSearch({pname}, {".ini"});
 
   wxArrayString strs;
   for (const std::string& filename : sv)

Source/Core/VideoCommon/HiresTextures.cpp

@@ -97,7 +97,7 @@ void HiresTexture::Update()
   };
 
   std::vector<std::string> filenames =
-      Common::DoFileSearch(extensions, {texture_directory}, /*recursive*/ true);
+      Common::DoFileSearch({texture_directory}, extensions, /*recursive*/ true);
 
   const std::string code = game_id + "_";
 

Source/Core/VideoCommon/PostProcessing.cpp

@@ -31,8 +31,9 @@ PostProcessingShaderImplementation::~PostProcessingShaderImplementation()
 static std::vector<std::string> GetShaders(const std::string& sub_dir = "")
 {
   std::vector<std::string> paths =
-      Common::DoFileSearch({".glsl"}, {File::GetUserPath(D_SHADERS_IDX) + sub_dir,
+      Common::DoFileSearch({File::GetUserPath(D_SHADERS_IDX) + sub_dir,
-                                       File::GetSysDirectory() + SHADERS_DIR DIR_SEP + sub_dir});
+                            File::GetSysDirectory() + SHADERS_DIR DIR_SEP + sub_dir},
+                           {".glsl"});
   std::vector<std::string> result;
   for (std::string path : paths)
   {