Merge pull request #8905 from JosJuice/jni-encoding

Android: Use correct encoding when converting strings

Source/Core/Common/FileUtil.cpp

@@ -114,7 +114,7 @@ bool Exists(const std::string& path)
 bool IsDirectory(const std::string& path)
 {
 #ifdef _WIN32
-  return PathIsDirectory(UTF8ToUTF16(path).c_str());
+  return PathIsDirectory(UTF8ToWString(path).c_str());
 #else
   return FileInfo(path).IsDirectory();
 #endif

Source/Core/Common/Logging/ConsoleListenerWin.cpp

@@ -17,5 +17,5 @@ ConsoleListener::~ConsoleListener()
 
 void ConsoleListener::Log([[maybe_unused]] Common::Log::LOG_LEVELS level, const char* text)
 {
-  ::OutputDebugStringW(UTF8ToUTF16(text).c_str());
+  ::OutputDebugStringW(UTF8ToWString(text).c_str());
 }

Source/Core/Common/StringUtil.cpp

@@ -5,6 +5,7 @@
 #include "Common/StringUtil.h"
 
 #include <algorithm>
+#include <codecvt>
 #include <cstdarg>
 #include <cstddef>
 #include <cstdio>
@@ -17,6 +18,7 @@
 #include <locale>
 #include <sstream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 #include <fmt/format.h>
@@ -32,7 +34,6 @@
 constexpr u32 CODEPAGE_SHIFT_JIS = 932;
 constexpr u32 CODEPAGE_WINDOWS_1252 = 1252;
 #else
-#include <codecvt>
 #include <errno.h>
 #include <iconv.h>
 #include <locale.h>
@@ -463,29 +464,29 @@ std::string UTF16ToCP(u32 code_page, std::wstring_view input)
   return output;
 }
 
-std::wstring UTF8ToUTF16(std::string_view input)
+std::wstring UTF8ToWString(std::string_view input)
 {
   return CPToUTF16(CP_UTF8, input);
 }
 
-std::string UTF16ToUTF8(std::wstring_view input)
+std::string WStringToUTF8(std::wstring_view input)
 {
   return UTF16ToCP(CP_UTF8, input);
 }
 
 std::string SHIFTJISToUTF8(std::string_view input)
 {
-  return UTF16ToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input));
+  return WStringToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input));
 }
 
 std::string UTF8ToSHIFTJIS(std::string_view input)
 {
-  return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToUTF16(input));
+  return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToWString(input));
 }
 
 std::string CP1252ToUTF8(std::string_view input)
 {
-  return UTF16ToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input));
+  return WStringToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input));
 }
 
 std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
@@ -493,7 +494,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
   const char16_t* str_end = std::find(str, str + max_size, '\0');
   std::wstring result(static_cast<size_t>(str_end - str), '\0');
   std::transform(str, str_end, result.begin(), static_cast<u16 (&)(u16)>(Common::swap16));
-  return UTF16ToUTF8(result);
+  return WStringToUTF8(result);
 }
 
 #else
@@ -578,9 +579,12 @@ std::string UTF8ToSHIFTJIS(std::string_view input)
   return CodeTo("SJIS", "UTF-8", input);
 }
 
-std::string UTF16ToUTF8(std::wstring_view input)
+std::string WStringToUTF8(std::wstring_view input)
 {
-  std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
+  using codecvt = std::conditional_t<sizeof(wchar_t) == 2, std::codecvt_utf8_utf16<wchar_t>,
+                                     std::codecvt_utf8<wchar_t>>;
+
+  std::wstring_convert<codecvt, wchar_t> converter;
   return converter.to_bytes(input.data(), input.data() + input.size());
 }
 
@@ -592,12 +596,24 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size)
 
 #endif
 
+std::string UTF16ToUTF8(std::u16string_view input)
+{
+  std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+  return converter.to_bytes(input.data(), input.data() + input.size());
+}
+
+std::u16string UTF8ToUTF16(std::string_view input)
+{
+  std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
+  return converter.from_bytes(input.data(), input.data() + input.size());
+}
+
 #ifdef HAS_STD_FILESYSTEM
 // This is a replacement for path::u8path, which is deprecated starting with C++20.
 std::filesystem::path StringToPath(std::string_view path)
 {
 #ifdef _MSC_VER
-  return std::filesystem::path(UTF8ToUTF16(path));
+  return std::filesystem::path(UTF8ToWString(path));
 #else
   return std::filesystem::path(path);
 #endif
@@ -608,7 +624,7 @@ std::filesystem::path StringToPath(std::string_view path)
 std::string PathToString(const std::filesystem::path& path)
 {
 #ifdef _MSC_VER
-  return UTF16ToUTF8(path.native());
+  return WStringToUTF8(path.native());
 #else
   return path.native();
 #endif

Source/Core/Common/StringUtil.h

@@ -172,22 +172,24 @@ size_t StringUTF8CodePointCount(const std::string& str);
 std::string CP1252ToUTF8(std::string_view str);
 std::string SHIFTJISToUTF8(std::string_view str);
 std::string UTF8ToSHIFTJIS(std::string_view str);
-std::string UTF16ToUTF8(std::wstring_view str);
+std::string WStringToUTF8(std::wstring_view str);
 std::string UTF16BEToUTF8(const char16_t* str, size_t max_size);  // Stops at \0
+std::string UTF16ToUTF8(std::u16string_view str);
+std::u16string UTF8ToUTF16(std::string_view str);
 
 #ifdef _WIN32
 
-std::wstring UTF8ToUTF16(std::string_view str);
+std::wstring UTF8ToWString(std::string_view str);
 
 #ifdef _UNICODE
 inline std::string TStrToUTF8(std::wstring_view str)
 {
-  return UTF16ToUTF8(str);
+  return WStringToUTF8(str);
 }
 
 inline std::wstring UTF8ToTStr(std::string_view str)
 {
-  return UTF8ToUTF16(str);
+  return UTF8ToWString(str);
 }
 #else
 inline std::string TStrToUTF8(std::string_view str)
@@ -221,7 +223,7 @@ std::string ThousandSeparate(I value, int spaces = 0)
   stream << std::setw(spaces) << value;
 
 #ifdef _WIN32
-  return UTF16ToUTF8(stream.str());
+  return WStringToUTF8(stream.str());
 #else
   return stream.str();
 #endif

Source/Core/Common/Timer.cpp

@@ -217,7 +217,7 @@ std::string Timer::GetTimeFormatted()
 #ifdef _WIN32
   struct timeb tp;
   (void)::ftime(&tp);
-  return UTF16ToUTF8(tmp) + fmt::format(":{:03}", tp.millitm);
+  return WStringToUTF8(tmp) + fmt::format(":{:03}", tp.millitm);
 #elif defined __APPLE__
   struct timeval t;
   (void)gettimeofday(&t, nullptr);
@@ -255,7 +255,7 @@ std::string Timer::GetDateTimeFormatted(double time)
 #ifdef _WIN32
   wchar_t tmp[32] = {};
   wcsftime(tmp, sizeof(tmp), L"%x %X", localTime);
-  return UTF16ToUTF8(tmp);
+  return WStringToUTF8(tmp);
 #else
   char tmp[32] = {};
   strftime(tmp, sizeof(tmp), "%x %X", localTime);