Import/Export signature files as CSV

This commit is contained in:
Sepalani 2016-12-06 15:43:41 +00:00
parent f431b18675
commit a6114bad34
16 changed files with 306 additions and 105 deletions

View File

@ -373,6 +373,16 @@ std::string ReplaceAll(std::string result, const std::string& src, const std::st
return result;
}
bool StringBeginsWith(const std::string& str, const std::string& begin)
{
return str.size() >= begin.size() && std::equal(begin.begin(), begin.end(), str.begin());
}
bool StringEndsWith(const std::string& str, const std::string& end)
{
return str.size() >= end.size() && std::equal(end.rbegin(), end.rend(), str.rbegin());
}
#ifdef _WIN32
std::string UTF16ToUTF8(const std::wstring& input)

View File

@ -116,6 +116,9 @@ void BuildCompleteFilename(std::string& _CompleteFilename, const std::string& _P
const std::string& _Filename);
std::string ReplaceAll(std::string result, const std::string& src, const std::string& dest);
bool StringBeginsWith(const std::string& str, const std::string& begin);
bool StringEndsWith(const std::string& str, const std::string& end);
std::string CP1252ToUTF8(const std::string& str);
std::string SHIFTJISToUTF8(const std::string& str);
std::string UTF16ToUTF8(const std::wstring& str);

View File

@ -31,7 +31,7 @@
#include "Core/PowerPC/PPCAnalyst.h"
#include "Core/PowerPC/PPCSymbolDB.h"
#include "Core/PowerPC/PowerPC.h"
#include "Core/PowerPC/SignatureDB.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
#include "DiscIO/Enums.h"
#include "DiscIO/NANDContentLoader.h"

View File

@ -163,7 +163,9 @@ set(SRCS ActionReplay.cpp
PowerPC/PPCSymbolDB.cpp
PowerPC/PPCTables.cpp
PowerPC/Profiler.cpp
PowerPC/SignatureDB.cpp
PowerPC/SignatureDB/CSVSignatureDB.cpp
PowerPC/SignatureDB/DSYSignatureDB.cpp
PowerPC/SignatureDB/SignatureDB.cpp
PowerPC/JitInterface.cpp
PowerPC/Interpreter/Interpreter_Branch.cpp
PowerPC/Interpreter/Interpreter.cpp

View File

@ -245,6 +245,9 @@
<ClCompile Include="PowerPC\JitCommon\JitAsmCommon.cpp" />
<ClCompile Include="PowerPC\JitCommon\JitBase.cpp" />
<ClCompile Include="PowerPC\JitCommon\JitCache.cpp" />
<ClCompile Include="PowerPC\SignatureDB\CSVSignatureDB.cpp" />
<ClCompile Include="PowerPC\SignatureDB\DSYSignatureDB.cpp" />
<ClCompile Include="PowerPC\SignatureDB\SignatureDB.cpp" />
<ClCompile Include="PowerPC\CachedInterpreter.cpp" />
<ClCompile Include="PowerPC\JitInterface.cpp" />
<ClCompile Include="PowerPC\MMU.cpp" />
@ -254,7 +257,6 @@
<ClCompile Include="PowerPC\PPCSymbolDB.cpp" />
<ClCompile Include="PowerPC\PPCTables.cpp" />
<ClCompile Include="PowerPC\Profiler.cpp" />
<ClCompile Include="PowerPC\SignatureDB.cpp" />
<ClCompile Include="State.cpp" />
</ItemGroup>
<ItemGroup>
@ -438,6 +440,9 @@
<ClInclude Include="PowerPC\JitCommon\JitAsmCommon.h" />
<ClInclude Include="PowerPC\JitCommon\JitBase.h" />
<ClInclude Include="PowerPC\JitCommon\JitCache.h" />
<ClInclude Include="PowerPC\SignatureDB\CSVSignatureDB.h" />
<ClInclude Include="PowerPC\SignatureDB\DSYSignatureDB.h" />
<ClInclude Include="PowerPC\SignatureDB\SignatureDB.h" />
<ClInclude Include="PowerPC\CachedInterpreter.h" />
<ClInclude Include="PowerPC\JitInterface.h" />
<ClInclude Include="PowerPC\PowerPC.h" />
@ -446,7 +451,6 @@
<ClInclude Include="PowerPC\PPCSymbolDB.h" />
<ClInclude Include="PowerPC\PPCTables.h" />
<ClInclude Include="PowerPC\Profiler.h" />
<ClInclude Include="PowerPC\SignatureDB.h" />
<ClInclude Include="State.h" />
</ItemGroup>
<ItemGroup>

View File

@ -648,9 +648,6 @@
<ClCompile Include="PowerPC\Profiler.cpp">
<Filter>PowerPC</Filter>
</ClCompile>
<ClCompile Include="PowerPC\SignatureDB.cpp">
<Filter>PowerPC</Filter>
</ClCompile>
<ClCompile Include="PowerPC\JitCommon\JitAsmCommon.cpp">
<Filter>PowerPC\JitCommon</Filter>
</ClCompile>
@ -754,6 +751,15 @@
<ClCompile Include="IPC_HLE\WII_IPC_HLE_Device_usb_ven.cpp">
<Filter>IPC HLE %28IOS/Starlet%29\USB</Filter>
</ClCompile>
<ClCompile Include="PowerPC\SignatureDB\CSVSignatureDB.cpp">
<Filter>PowerPC\SignatureDB</Filter>
</ClCompile>
<ClCompile Include="PowerPC\SignatureDB\DSYSignatureDB.cpp">
<Filter>PowerPC\SignatureDB</Filter>
</ClCompile>
<ClCompile Include="PowerPC\SignatureDB\SignatureDB.cpp">
<Filter>PowerPC\SignatureDB</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="BootManager.h" />
@ -1238,9 +1244,6 @@
<ClInclude Include="PowerPC\Profiler.h">
<Filter>PowerPC</Filter>
</ClInclude>
<ClInclude Include="PowerPC\SignatureDB.h">
<Filter>PowerPC</Filter>
</ClInclude>
<ClInclude Include="PowerPC\JitCommon\JitAsmCommon.h">
<Filter>PowerPC\JitCommon</Filter>
</ClInclude>
@ -1294,6 +1297,15 @@
<ClInclude Include="IPC_HLE\WII_IPC_HLE_Device_usb_ven.h">
<Filter>IPC HLE %28IOS/Starlet%29\USB</Filter>
</ClInclude>
<ClInclude Include="PowerPC\SignatureDB\CSVSignatureDB.h">
<Filter>PowerPC\SignatureDB</Filter>
</ClInclude>
<ClInclude Include="PowerPC\SignatureDB\DSYSignatureDB.h">
<Filter>PowerPC\SignatureDB</Filter>
</ClInclude>
<ClInclude Include="PowerPC\SignatureDB\SignatureDB.h">
<Filter>PowerPC\SignatureDB</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="CMakeLists.txt" />

View File

@ -15,7 +15,7 @@
#include "Core/PowerPC/PPCSymbolDB.h"
#include "Core/PowerPC/PPCTables.h"
#include "Core/PowerPC/PowerPC.h"
#include "Core/PowerPC/SignatureDB.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
// Analyzes PowerPC code in memory to find functions
// After running, for each function we will know what functions it calls

View File

@ -14,7 +14,7 @@
#include "Core/PowerPC/PPCAnalyst.h"
#include "Core/PowerPC/PPCSymbolDB.h"
#include "Core/PowerPC/PowerPC.h"
#include "Core/PowerPC/SignatureDB.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
static std::string GetStrippedFunctionName(const std::string& symbol_name)
{

View File

@ -0,0 +1,75 @@
#include <cstdio>
#include <fstream>
#include <sstream>
#include "Common/FileUtil.h"
#include "Common/Logging/Log.h"
#include "Core/PowerPC/SignatureDB/CSVSignatureDB.h"
// CSV separated with tabs
// Checksum | Size | Symbol | [Object Location |] Object Name
bool CSVSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& database) const
{
std::string line;
std::ifstream ifs;
OpenFStream(ifs, file_path, std::ios_base::in);
if (!ifs)
return false;
for (size_t i = 1; std::getline(ifs, line); i += 1)
{
std::istringstream iss(line);
u32 checksum, size;
std::string tab, symbol, object_location, object_name;
iss >> std::hex >> checksum >> std::hex >> size;
if (iss && std::getline(iss, tab, '\t'))
{
if (std::getline(iss, symbol, '\t') && std::getline(iss, object_location, '\t'))
std::getline(iss, object_name);
SignatureDB::DBFunc func;
func.name = symbol;
func.size = size;
// Doesn't have an object location
if (object_name.empty())
{
func.object_name = object_location;
}
else
{
func.object_location = object_location;
func.object_name = object_name;
}
database[checksum] = func;
}
else
{
WARN_LOG(OSHLE, "CSV database failed to parse line %zu", i);
}
}
return true;
}
bool CSVSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncDB& database) const
{
File::IOFile f(file_path, "w");
if (!f)
{
ERROR_LOG(OSHLE, "CSV database save failed");
return false;
}
for (const auto& func : database)
{
// The object name/location are unused for the time being.
// To be implemented.
fprintf(f.GetHandle(), "%08x\t%08x\t%s\t%s\t%s\n", func.first, func.second.size,
func.second.name.c_str(), func.second.object_location.c_str(),
func.second.object_name.c_str());
}
INFO_LOG(OSHLE, "CSV database save successful");
return true;
}

View File

@ -0,0 +1,11 @@
#pragma once
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
class CSVSignatureDB final : public SignatureDBFormatHandler
{
public:
~CSVSignatureDB() = default;
bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const override;
bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const override;
};

View File

@ -0,0 +1,69 @@
#include <cstddef>
#include <cstring>
#include "Common/CommonTypes.h"
#include "Common/FileUtil.h"
#include "Common/Logging/Log.h"
#include "Core/PowerPC/SignatureDB/DSYSignatureDB.h"
namespace
{
// On-disk format for SignatureDB entries.
struct FuncDesc
{
u32 checksum;
u32 size;
char name[128];
};
} // namespace
bool DSYSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& database) const
{
File::IOFile f(file_path, "rb");
if (!f)
return false;
u32 fcount = 0;
f.ReadArray(&fcount, 1);
for (size_t i = 0; i < fcount; i++)
{
FuncDesc temp;
memset(&temp, 0, sizeof(temp));
f.ReadArray(&temp, 1);
temp.name[sizeof(temp.name) - 1] = 0;
SignatureDB::DBFunc func;
func.name = temp.name;
func.size = temp.size;
database[temp.checksum] = func;
}
return true;
}
bool DSYSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncDB& database) const
{
File::IOFile f(file_path, "wb");
if (!f)
{
ERROR_LOG(OSHLE, "Database save failed");
return false;
}
u32 fcount = static_cast<u32>(database.size());
f.WriteArray(&fcount, 1);
for (const auto& entry : database)
{
FuncDesc temp;
memset(&temp, 0, sizeof(temp));
temp.checksum = entry.first;
temp.size = entry.second.size;
strncpy(temp.name, entry.second.name.c_str(), 127);
f.WriteArray(&temp, 1);
}
INFO_LOG(OSHLE, "Database save successful");
return true;
}

View File

@ -0,0 +1,11 @@
#pragma once
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
class DSYSignatureDB final : public SignatureDBFormatHandler
{
public:
~DSYSignatureDB() = default;
bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const override;
bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const override;
};

View File

@ -7,69 +7,34 @@
#include "Common/CommonTypes.h"
#include "Common/FileUtil.h"
#include "Common/Logging/Log.h"
#include "Common/StringUtil.h"
#include "Core/PowerPC/PPCAnalyst.h"
#include "Core/PowerPC/PPCSymbolDB.h"
#include "Core/PowerPC/PowerPC.h"
#include "Core/PowerPC/SignatureDB.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
namespace
// Format Handlers
#include "Core/PowerPC/SignatureDB/CSVSignatureDB.h"
#include "Core/PowerPC/SignatureDB/DSYSignatureDB.h"
std::unique_ptr<SignatureDBFormatHandler>
SignatureDB::CreateFormatHandler(const std::string& file_path)
{
// On-disk format for SignatureDB entries.
struct FuncDesc
{
u32 checkSum;
u32 size;
char name[128];
};
} // namespace
bool SignatureDB::Load(const std::string& filename)
{
File::IOFile f(filename, "rb");
if (!f)
return false;
u32 fcount = 0;
f.ReadArray(&fcount, 1);
for (size_t i = 0; i < fcount; i++)
{
FuncDesc temp;
memset(&temp, 0, sizeof(temp));
f.ReadArray(&temp, 1);
temp.name[sizeof(temp.name) - 1] = 0;
DBFunc dbf;
dbf.name = temp.name;
dbf.size = temp.size;
database[temp.checkSum] = dbf;
}
return true;
if (StringEndsWith(file_path, ".csv"))
return std::make_unique<CSVSignatureDB>();
return std::make_unique<DSYSignatureDB>();
}
bool SignatureDB::Save(const std::string& filename)
bool SignatureDB::Load(const std::string& file_path)
{
File::IOFile f(filename, "wb");
if (!f)
{
ERROR_LOG(OSHLE, "Database save failed");
return false;
}
u32 fcount = (u32)database.size();
f.WriteArray(&fcount, 1);
for (const auto& entry : database)
{
FuncDesc temp;
memset(&temp, 0, sizeof(temp));
temp.checkSum = entry.first;
temp.size = entry.second.size;
strncpy(temp.name, entry.second.name.c_str(), 127);
f.WriteArray(&temp, 1);
}
auto handler = CreateFormatHandler(file_path);
return handler->Load(file_path, m_database);
}
INFO_LOG(OSHLE, "Database save successful");
return true;
bool SignatureDB::Save(const std::string& file_path)
{
auto handler = CreateFormatHandler(file_path);
return handler->Save(file_path, m_database);
}
// Adds a known function to the hash database
@ -81,31 +46,31 @@ u32 SignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
temp_dbfunc.size = size;
temp_dbfunc.name = name;
FuncDB::iterator iter = database.find(hash);
if (iter == database.end())
database[hash] = temp_dbfunc;
FuncDB::iterator iter = m_database.find(hash);
if (iter == m_database.end())
m_database[hash] = temp_dbfunc;
return hash;
}
void SignatureDB::List()
{
for (const auto& entry : database)
for (const auto& entry : m_database)
{
DEBUG_LOG(OSHLE, "%s : %i bytes, hash = %08x", entry.second.name.c_str(), entry.second.size,
entry.first);
}
INFO_LOG(OSHLE, "%zu functions known in current database.", database.size());
INFO_LOG(OSHLE, "%zu functions known in current database.", m_database.size());
}
void SignatureDB::Clear()
{
database.clear();
m_database.clear();
}
void SignatureDB::Apply(PPCSymbolDB* symbol_db)
{
for (const auto& entry : database)
for (const auto& entry : m_database)
{
u32 hash = entry.first;
Symbol* function = symbol_db->GetSymbolFromHash(hash);
@ -140,7 +105,7 @@ void SignatureDB::Initialize(PPCSymbolDB* symbol_db, const std::string& prefix)
DBFunc temp_dbfunc;
temp_dbfunc.name = symbol.second.name;
temp_dbfunc.size = symbol.second.size;
database[symbol.second.hash] = temp_dbfunc;
m_database[symbol.second.hash] = temp_dbfunc;
}
}
}
@ -203,3 +168,7 @@ void SignatureDB::Initialize(PPCSymbolDB* symbol_db, const std::string& prefix)
}
return sum;
}
SignatureDBFormatHandler::~SignatureDBFormatHandler()
{
}

View File

@ -5,6 +5,7 @@
#pragma once
#include <map>
#include <memory>
#include <string>
#include "Common/CommonTypes.h"
@ -12,28 +13,27 @@
// You're not meant to keep around SignatureDB objects persistently. Use 'em, throw them away.
class PPCSymbolDB;
class SignatureDBFormatHandler;
class SignatureDB
{
public:
struct DBFunc
{
std::string name;
u32 size;
std::string name;
std::string object_name;
std::string object_location;
DBFunc() : size(0) {}
};
using FuncDB = std::map<u32, DBFunc>;
// Map from signature to function. We store the DB in this map because it optimizes the
// most common operation - lookup. We don't care about ordering anyway.
typedef std::map<u32, DBFunc> FuncDB;
FuncDB database;
public:
// Returns the hash.
u32 Add(u32 startAddr, u32 size, const std::string& name);
bool Load(const std::string&
filename); // Does not clear. Remember to clear first if that's what you want.
bool Save(const std::string& filename);
// Does not clear. Remember to clear first if that's what you want.
bool Load(const std::string& file_path);
bool Save(const std::string& file_path);
void Clear();
void List();
@ -41,4 +41,18 @@ public:
void Apply(PPCSymbolDB* func_db);
static u32 ComputeCodeChecksum(u32 offsetStart, u32 offsetEnd);
private:
std::unique_ptr<SignatureDBFormatHandler> CreateFormatHandler(const std::string& file_path);
// Map from signature to function. We store the DB in this map because it optimizes the
// most common operation - lookup. We don't care about ordering anyway.
FuncDB m_database;
};
class SignatureDBFormatHandler
{
public:
virtual ~SignatureDBFormatHandler();
virtual bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const = 0;
virtual bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const = 0;
};

View File

@ -32,7 +32,7 @@
#include "Core/PowerPC/PPCSymbolDB.h"
#include "Core/PowerPC/PowerPC.h"
#include "Core/PowerPC/Profiler.h"
#include "Core/PowerPC/SignatureDB.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
#include "DolphinWX/Debugger/BreakpointWindow.h"
#include "DolphinWX/Debugger/CodeWindow.h"
@ -161,6 +161,9 @@ void CCodeWindow::OnProfilerMenu(wxCommandEvent& event)
void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
{
static const wxString signature_selector = _("Dolphin Signature File (*.dsy)") + "|*.dsy|" +
_("Dolphin Signature CSV File (*.csv)") + "|*.csv|" +
wxGetTranslation(wxALL_FILES);
Parent->ClearStatusBar();
if (!Core::IsRunning())
@ -309,8 +312,7 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
std::string prefix(WxStrToStr(input_prefix.GetValue()));
wxString path = wxFileSelector(_("Save signature as"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, _("Dolphin Signature File (*.dsy)") +
"|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxEmptyString, signature_selector,
wxFD_SAVE | wxFD_OVERWRITE_PROMPT, this);
if (!path.IsEmpty())
{
@ -332,10 +334,9 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
{
std::string prefix(WxStrToStr(input_prefix.GetValue()));
wxString path = wxFileSelector(
_("Append signature to"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
_("Dolphin Signature File (*.dsy)") + "|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxFD_SAVE, this);
wxString path =
wxFileSelector(_("Append signature to"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, signature_selector, wxFD_SAVE, this);
if (!path.IsEmpty())
{
SignatureDB db;
@ -350,10 +351,9 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
break;
case IDM_USE_SIGNATURE_FILE:
{
wxString path = wxFileSelector(
_("Apply signature file"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
_("Dolphin Signature File (*.dsy)") + "|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
wxString path =
wxFileSelector(_("Apply signature file"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path.IsEmpty())
{
SignatureDB db;
@ -366,25 +366,22 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
break;
case IDM_COMBINE_SIGNATURE_FILES:
{
wxString path1 = wxFileSelector(
_("Choose priority input file"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
_("Dolphin Signature File (*.dsy)") + "|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
wxString path1 =
wxFileSelector(_("Choose priority input file"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path1.IsEmpty())
{
SignatureDB db;
wxString path2 = wxFileSelector(
_("Choose secondary input file"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
_("Dolphin Signature File (*.dsy)") + "|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
wxString path2 =
wxFileSelector(_("Choose secondary input file"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path2.IsEmpty())
{
db.Load(WxStrToStr(path2));
db.Load(WxStrToStr(path1));
path2 = wxFileSelector(_("Save combined output file as"), File::GetSysDirectory(),
wxEmptyString, ".dsy", _("Dolphin Signature File (*.dsy)") +
"|*.dsy|" + wxGetTranslation(wxALL_FILES),
wxEmptyString, ".dsy", signature_selector,
wxFD_SAVE | wxFD_OVERWRITE_PROMPT, this);
db.Save(WxStrToStr(path2));
db.List();

View File

@ -16,3 +16,27 @@ TEST(StringUtil, JoinStrings)
EXPECT_EQ("a, bb, c", JoinStrings({"a", "bb", "c"}, ", "));
EXPECT_EQ("???", JoinStrings({"?", "?"}, "?"));
}
TEST(StringUtil, StringBeginsWith)
{
EXPECT_EQ(true, StringBeginsWith("abc", "a"));
EXPECT_EQ(false, StringBeginsWith("abc", "b"));
EXPECT_EQ(true, StringBeginsWith("abc", "ab"));
EXPECT_EQ(false, StringBeginsWith("a", "ab"));
EXPECT_EQ(false, StringBeginsWith("", "a"));
EXPECT_EQ(false, StringBeginsWith("", "ab"));
EXPECT_EQ(true, StringBeginsWith("abc", ""));
EXPECT_EQ(true, StringBeginsWith("", ""));
}
TEST(StringUtil, StringEndsWith)
{
EXPECT_EQ(true, StringEndsWith("abc", "c"));
EXPECT_EQ(false, StringEndsWith("abc", "b"));
EXPECT_EQ(true, StringEndsWith("abc", "bc"));
EXPECT_EQ(false, StringEndsWith("a", "ab"));
EXPECT_EQ(false, StringEndsWith("", "a"));
EXPECT_EQ(false, StringEndsWith("", "ab"));
EXPECT_EQ(true, StringEndsWith("abc", ""));
EXPECT_EQ(true, StringEndsWith("", ""));
}