Files
melonDS/src/FATStorage.cpp

955 lines
25 KiB
C++

/*
Copyright 2016-2021 Arisotura
This file is part of melonDS.
melonDS is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation, either version 3 of the License, or (at your option)
any later version.
melonDS is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with melonDS. If not, see http://www.gnu.org/licenses/.
*/
#ifdef __WIN32__
#include <windows.h>
#else
#include <sys/types.h>
#endif // __WIN32__
#include <dirent.h>
#include <errno.h>
#include <inttypes.h>
#include "FATStorage.h"
#include "Platform.h"
namespace fs = std::filesystem;
static int GetDirEntryType(const char* path, struct dirent* entry)
{
#ifdef __WIN32__
DWORD res = GetFileAttributesA(path);
if (res == INVALID_FILE_ATTRIBUTES) return -1;
if (res & FILE_ATTRIBUTE_DIRECTORY) return 1;
if (res & (FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DEVICE | FILE_ATTRIBUTE_VIRTUAL)) return -1;
return 0;
#else
if (entry->d_type == DT_DIR) return 1;
if (entry->d_type == DT_REG) return 0;
return -1;
#endif // __WIN32__
}
FATStorage::FATStorage()
{
printf("FATStorage begin\n");
bool res = Build("dldi", 0x20000000, "melonDLDI.bin");
printf("FATStorage result: %d\n", res);
}
FATStorage::~FATStorage()
{
printf("SAVING DLDI SHIT\n");
Save("dldi");
}
FILE* FATStorage::FF_File;
u64 FATStorage::FF_FileSize;
UINT FATStorage::FF_ReadStorage(BYTE* buf, LBA_t sector, UINT num)
{
if (!FF_File) return 0;
u64 addr = sector * 0x200ULL;
u32 len = num * 0x200;
if ((addr+len) > FF_FileSize)
{
if (addr >= FF_FileSize) return 0;
len = FF_FileSize - addr;
num = len >> 9;
}
fseek(FF_File, addr, SEEK_SET);
u32 res = fread(buf, 0x200, num, FF_File);
if (res < num)
{
if (feof(FF_File))
{
memset(&buf[0x200*res], 0, 0x200*(num-res));
return num;
}
}
return res;
}
UINT FATStorage::FF_WriteStorage(BYTE* buf, LBA_t sector, UINT num)
{
if (!FF_File) return 0;
u64 addr = sector * 0x200ULL;
u32 len = num * 0x200;
if ((addr+len) > FF_FileSize)
{
if (addr >= FF_FileSize) return 0;
len = FF_FileSize - addr;
num = len >> 9;
}
fseek(FF_File, addr, SEEK_SET);
u32 res = fwrite(buf, 0x200, num, FF_File);
return res;
}
void FATStorage::LoadIndex()
{
DirIndex.clear();
FileIndex.clear();
FILE* f = Platform::OpenLocalFile(IndexPath.c_str(), "r");
if (!f) return;
char linebuf[1536];
while (!feof(f))
{
if (fgets(linebuf, 1536, f) == nullptr)
break;
if (linebuf[0] == 'D')
{
u32 readonly;
char fpath[1536] = {0};
int ret = sscanf(linebuf, "DIR %u %[^\t\r\n]",
&readonly, fpath);
if (ret < 2) continue;
for (int i = 0; i < 1536 && fpath[i] != '\0'; i++)
{
if (fpath[i] == '\\')
fpath[i] = '/';
}
DirIndexEntry entry;
entry.Path = fpath;
entry.IsReadOnly = readonly!=0;
DirIndex[entry.Path] = entry;
}
else if (linebuf[0] == 'F')
{
u32 readonly;
u64 fsize;
s64 lastmodified;
u32 lastmod_internal;
char fpath[1536] = {0};
int ret = sscanf(linebuf, "FILE %u %" PRIu64 " %" PRId64 " %u %[^\t\r\n]",
&readonly, &fsize, &lastmodified, &lastmod_internal, fpath);
if (ret < 5) continue;
for (int i = 0; i < 1536 && fpath[i] != '\0'; i++)
{
if (fpath[i] == '\\')
fpath[i] = '/';
}
FileIndexEntry entry;
entry.Path = fpath;
entry.IsReadOnly = readonly!=0;
entry.Size = fsize;
entry.LastModified = lastmodified;
entry.LastModifiedInternal = lastmod_internal;
FileIndex[entry.Path] = entry;
}
}
fclose(f);
// ensure the indexes are sane
std::vector<std::string> removelist;
for (const auto& [key, val] : DirIndex)
{
std::string path = val.Path;
if ((path.find("/./") != std::string::npos) ||
(path.find("/../") != std::string::npos) ||
(path.substr(0,2) == "./") ||
(path.substr(0,3) == "../"))
{
removelist.push_back(key);
continue;
}
int sep = path.rfind('/');
if (sep == std::string::npos) continue;
path = path.substr(0, sep);
if (DirIndex.count(path) < 1)
{
removelist.push_back(key);
}
}
for (const auto& key : removelist)
{
DirIndex.erase(key);
}
removelist.clear();
for (const auto& [key, val] : FileIndex)
{
std::string path = val.Path;
if ((path.find("/./") != std::string::npos) ||
(path.find("/../") != std::string::npos) ||
(path.substr(0,2) == "./") ||
(path.substr(0,3) == "../"))
{
removelist.push_back(key);
continue;
}
int sep = path.rfind('/');
if (sep == std::string::npos) continue;
path = path.substr(0, sep);
if (DirIndex.count(path) < 1)
{
removelist.push_back(key);
}
}
for (const auto& key : removelist)
{
FileIndex.erase(key);
}
}
void FATStorage::SaveIndex()
{
FILE* f = Platform::OpenLocalFile(IndexPath.c_str(), "w");
if (!f) return;
for (const auto& [key, val] : DirIndex)
{
fprintf(f, "DIR %u %s\r\n",
val.IsReadOnly?1:0, val.Path.c_str());
}
for (const auto& [key, val] : FileIndex)
{
fprintf(f, "FILE %u %" PRIu64 " %" PRId64 " %u %s\r\n",
val.IsReadOnly?1:0, val.Size, val.LastModified, val.LastModifiedInternal, val.Path.c_str());
}
fclose(f);
}
bool FATStorage::ExportFile(std::string path, std::string out, fs::file_time_type& modtime)
{
FF_FIL file;
FILE* fout;
FRESULT res;
res = f_open(&file, path.c_str(), FA_OPEN_EXISTING | FA_READ);
if (res != FR_OK)
return false;
u32 len = f_size(&file);
if (fs::exists(out))
{
std::error_code err;
fs::permissions(out,
fs::perms::owner_read | fs::perms::owner_write,
fs::perm_options::add,
err);
}
fout = fopen(out.c_str(), "wb");
if (!fout)
{
f_close(&file);
return false;
}
u8 buf[0x1000];
for (u32 i = 0; i < len; i += 0x1000)
{
u32 blocklen;
if ((i + 0x1000) > len)
blocklen = len - i;
else
blocklen = 0x1000;
u32 nread;
f_read(&file, buf, blocklen, &nread);
fwrite(buf, blocklen, 1, fout);
}
fclose(fout);
f_close(&file);
modtime = fs::last_write_time(out);
return true;
}
void FATStorage::ExportDirectory(std::string path, std::string outbase, int level)
{
if (level >= 32) return;
FF_DIR dir;
FF_FILINFO info;
FRESULT res;
printf("EXPORTING DIRECTORY %s (base %s level %d)\n", path.c_str(), outbase.c_str(), level);
std::string fullpath = "0:/" + path;
res = f_opendir(&dir, fullpath.c_str());
if (res != FR_OK) return;
std::vector<std::string> subdirlist;
for (;;)
{
res = f_readdir(&dir, &info);
if (res != FR_OK) break;
if (!info.fname[0]) break;
std::string fullpath = path + info.fname;
if (info.fattrib & AM_DIR)
{
if (DirIndex.count(fullpath) < 1)
{
std::string dirpath = outbase+"/"+fullpath;
mkdir(dirpath.c_str());
DirIndexEntry entry;
entry.Path = fullpath;
entry.IsReadOnly = (info.fattrib & AM_RDO) != 0;
DirIndex[entry.Path] = entry;
}
subdirlist.push_back(fullpath);
}
else
{
bool doexport = false;
printf("- FILE %s ATTRIB %08X\n", fullpath.c_str(), info.fattrib);
if (FileIndex.count(fullpath) < 1)
{
doexport = true;
FileIndexEntry entry;
entry.Path = fullpath;
entry.IsReadOnly = (info.fattrib & AM_RDO) != 0;
entry.Size = info.fsize;
entry.LastModifiedInternal = (info.fdate << 16) | info.ftime;
FileIndex[entry.Path] = entry;
}
else
{
u32 lastmod = (info.fdate << 16) | info.ftime;
FileIndexEntry& entry = FileIndex[fullpath];
if ((info.fsize != entry.Size) || (lastmod != entry.LastModifiedInternal))
doexport = true;
entry.Size = info.fsize;
entry.LastModifiedInternal = lastmod;
}
if (doexport)
{
printf("EXPORTING FILE 0:/%s TO %s/%s\n", fullpath.c_str(), outbase.c_str(), fullpath.c_str());
fs::file_time_type modtime;
if (ExportFile("0:/"+fullpath, outbase+"/"+fullpath, modtime))
{
s64 modtime_raw = std::chrono::duration_cast<std::chrono::seconds>(modtime.time_since_epoch()).count();
FileIndexEntry& entry = FileIndex[fullpath];
entry.LastModified = modtime_raw;
}
else
{
// ??????
}
}
}
std::error_code err;
fs::permissions(outbase+"/"+fullpath,
fs::perms::owner_read | fs::perms::owner_write,
(info.fattrib & AM_RDO) ? fs::perm_options::remove : fs::perm_options::add,
err);
}
f_closedir(&dir);
for (auto& entry : subdirlist)
{
ExportDirectory(entry+"/", outbase, level+1);
}
}
bool FATStorage::DeleteHostDirectory(std::string path, std::string outbase, int level)
{
if (level >= 32) return false;
std::vector<std::string> filedeletelist;
std::vector<std::string> dirdeletelist;
printf("-- PURGING HOST DIRECTORY %s (base %s) LEVEL=%d\n", path.c_str(), outbase.c_str(), level);
int outlen = outbase.length();
for (auto& entry : fs::directory_iterator(outbase + "/" + path))
{
std::string fullpath = entry.path().string();
std::string innerpath = fullpath.substr(outlen);
if (innerpath[0] == '/' || innerpath[0] == '\\')
innerpath = innerpath.substr(1);
int ilen = innerpath.length();
for (int i = 0; i < ilen; i++)
{
if (innerpath[i] == '\\')
innerpath[i] = '/';
}
printf("---- ENTRY %s (%d)\n", innerpath.c_str(), entry.is_directory());
if (entry.is_directory())
{
dirdeletelist.push_back(innerpath);
}
else
{
filedeletelist.push_back(innerpath);
}
}
for (const auto& key : filedeletelist)
{
std::string fullpath = outbase + "/" + key;
std::error_code err;
fs::permissions(fullpath,
fs::perms::owner_read | fs::perms::owner_write,
fs::perm_options::add,
err);
if (!fs::remove(fullpath))
return false;
FileIndex.erase(key);
}
for (const auto& key : dirdeletelist)
{
if (!DeleteHostDirectory(key, outbase, level+1))
return false;
}
{
std::string fullpath = outbase + "/" + path;
printf("DELETING DIR ITSELF: %s\n", fullpath.c_str());
std::error_code err;
fs::permissions(fullpath,
fs::perms::owner_read | fs::perms::owner_write,
fs::perm_options::add,
err);
if (!fs::remove(fullpath)){printf("DICKPILE %d\n", errno);
return false;}
DirIndex.erase(path);
}
return true;
}
void FATStorage::ExportChanges(std::string outbase)
{
// reflect changes in the FAT volume to the host filesystem
// * delete directories and files that exist in the index but not in the volume
// * copy files to the host FS if they exist within the index and their size or
// internal last-modified time is different
// * index and copy directories and files that exist in the volume but not in
// the index
std::vector<std::string> deletelist;
for (const auto& [key, val] : FileIndex)
{
std::string innerpath = "0:/" + val.Path;
FF_FILINFO finfo;
FRESULT res = f_stat(innerpath.c_str(), &finfo);
if (res == FR_OK)
{
if (finfo.fattrib & AM_DIR)
{
deletelist.push_back(key);
}
}
else if (res == FR_NO_FILE || res == FR_NO_PATH)
{
deletelist.push_back(key);
}
}
for (const auto& key : deletelist)
{
std::string fullpath = outbase + "/" + key;
printf("DELETING FILE %s\n", fullpath.c_str());
std::error_code err;
fs::permissions(fullpath,
fs::perms::owner_read | fs::perms::owner_write,
fs::perm_options::add,
err);
fs::remove(fullpath);
FileIndex.erase(key);
}
deletelist.clear();
for (const auto& [key, val] : DirIndex)
{
std::string innerpath = "0:/" + val.Path;
FF_FILINFO finfo;
FRESULT res = f_stat(innerpath.c_str(), &finfo);
if (res == FR_OK)
{
if (!(finfo.fattrib & AM_DIR))
{
deletelist.push_back(key);
}
}
else if (res == FR_NO_FILE || res == FR_NO_PATH)
{
deletelist.push_back(key);
}
}
for (const auto& key : deletelist)
{
printf("DELETING DIRECTORY %s\n", key.c_str());
DeleteHostDirectory(key, outbase, 0);
}
ExportDirectory("", outbase, 0);
}
bool FATStorage::CanFitFile(u32 len)
{
FATFS* fs;
DWORD freeclusters;
FRESULT res;
res = f_getfree("0:", &freeclusters, &fs);
if (res != FR_OK) return false;
u32 clustersize = fs->csize * 0x200;
printf("CHECK IF FILE CAN FIT: len=%d, clus=%d, num=%d, free=%d\n",
len, clustersize, (len + clustersize - 1) / clustersize, freeclusters);
len = (len + clustersize - 1) / clustersize;
return (freeclusters >= len);
}
bool FATStorage::DeleteDirectory(std::string path, int level)
{
if (level >= 32) return false;
if (path.length() < 1) return false;
FF_DIR dir;
FF_FILINFO info;
FRESULT res;
std::string fullpath = "0:/" + path;
f_chmod(fullpath.c_str(), 0, AM_RDO);
res = f_opendir(&dir, fullpath.c_str());
if (res != FR_OK) return false;
std::vector<std::string> deletelist;
std::vector<std::string> subdirlist;
int survivors = 0;
for (;;)
{
res = f_readdir(&dir, &info);
if (res != FR_OK) break;
if (!info.fname[0]) break;
std::string fullpath = path + info.fname;
if (info.fattrib & AM_DIR)
{
subdirlist.push_back(fullpath);
}
else
{
deletelist.push_back(fullpath);
}
}
f_closedir(&dir);
for (auto& entry : deletelist)
{
std::string fullpath = "0:/" + entry;
f_chmod(fullpath.c_str(), 0, AM_RDO);
res = f_unlink(fullpath.c_str());
if (res != FR_OK) return false;
}
for (auto& entry : subdirlist)
{
if (!DeleteDirectory(entry+"/", level+1)) return false;
std::string fullpath = "0:/" + entry;
f_chmod(fullpath.c_str(), 0, AM_RDO);
res = f_unlink(fullpath.c_str());
if (res != FR_OK) return false;
}
res = f_unlink(fullpath.c_str());
if (res != FR_OK) return false;
return true;
}
void FATStorage::CleanupDirectory(std::string sourcedir, std::string path, int level)
{
if (level >= 32) return;
FF_DIR dir;
FF_FILINFO info;
FRESULT res;
printf("CLEANING UP DIRECTORY %s (level=%d)\n", path.c_str(), level);
std::string fullpath = "0:/" + path;
res = f_opendir(&dir, fullpath.c_str());
if (res != FR_OK) return;
std::vector<std::string> filedeletelist;
std::vector<std::string> dirdeletelist;
std::vector<std::string> subdirlist;
for (;;)
{
res = f_readdir(&dir, &info);
if (res != FR_OK) break;
if (!info.fname[0]) break;
std::string fullpath = path + info.fname;
printf("FOUND ENTRY %s %08X (%d/%d, %d)\n",
fullpath.c_str(), info.fattrib, FileIndex.count(fullpath), DirIndex.count(fullpath),
fs::exists(sourcedir+"/"+fullpath));
if (info.fattrib & AM_DIR)
{
if (DirIndex.count(fullpath) < 1)
dirdeletelist.push_back(fullpath);
else if (!fs::is_directory(sourcedir+"/"+fullpath))
{
DirIndex.erase(fullpath);
dirdeletelist.push_back(fullpath);
}
else
subdirlist.push_back(fullpath);
}
else
{
if (FileIndex.count(fullpath) < 1)
filedeletelist.push_back(fullpath);
else if (!fs::is_regular_file(sourcedir+"/"+fullpath))
{
FileIndex.erase(fullpath);
filedeletelist.push_back(fullpath);
}
}
}
f_closedir(&dir);
for (auto& entry : filedeletelist)
{
std::string fullpath = "0:/" + entry;
f_chmod(fullpath.c_str(), 0, AM_RDO);
f_unlink(fullpath.c_str());
}
for (auto& entry : dirdeletelist)
{
DeleteDirectory(entry+"/", level+1);
}
for (auto& entry : subdirlist)
{
CleanupDirectory(sourcedir, entry+"/", level+1);
}
}
bool FATStorage::ImportFile(std::string path, std::string in)
{
FF_FIL file;
FILE* fin;
FRESULT res;
fin = fopen(in.c_str(), "rb");
if (!fin)
return false;
fseek(fin, 0, SEEK_END);
u32 len = (u32)ftell(fin);
fseek(fin, 0, SEEK_SET);
if (!CanFitFile(len))
{
fclose(fin);
return false;
}
res = f_open(&file, path.c_str(), FA_CREATE_ALWAYS | FA_WRITE);
if (res != FR_OK)
{
fclose(fin);
return false;
}
u8 buf[0x1000];
for (u32 i = 0; i < len; i += 0x1000)
{
u32 blocklen;
if ((i + 0x1000) > len)
blocklen = len - i;
else
blocklen = 0x1000;
u32 nwrite;
fread(buf, blocklen, 1, fin);
f_write(&file, buf, blocklen, &nwrite);
}
fclose(fin);
f_close(&file);
return true;
}
bool FATStorage::BuildSubdirectory(const char* sourcedir, const char* path, int level)
{
if (level >= 32)
{
printf("FATStorage::BuildSubdirectory: too many subdirectory levels, skipping\n");
return false;
}
if (level == 0)
{
// remove whatever isn't in the index
CleanupDirectory(sourcedir, "", 0);
int srclen = strlen(sourcedir);
// iterate through the host directory:
// * directories will be added if they aren't in the index
// * files will be added if they aren't in the index, or if the size or last-modified-date don't match
for (auto& entry : fs::recursive_directory_iterator(sourcedir))
{
std::string fullpath = entry.path().string();
std::string innerpath = fullpath.substr(srclen);
if (innerpath[0] == '/' || innerpath[0] == '\\')
innerpath = innerpath.substr(1);
int ilen = innerpath.length();
for (int i = 0; i < ilen; i++)
{
if (innerpath[i] == '\\')
innerpath[i] = '/';
}
bool readonly = (entry.status().permissions() & fs::perms::owner_write) == fs::perms::none;
//std::chrono::duration<s64> bourf(lastmodified_raw);
//printf("DORP: %016llX\n", bourf.count());
if (entry.is_directory())
{
if (DirIndex.count(innerpath) < 1)
{
DirIndexEntry ientry;
ientry.Path = innerpath;
ientry.IsReadOnly = readonly;
innerpath = "0:/" + innerpath;
FRESULT res = f_mkdir(innerpath.c_str());
if (res == FR_OK)
{
DirIndex[ientry.Path] = ientry;
printf("IMPORTING DIR %s (FROM %s), %d\n", innerpath.c_str(), fullpath.c_str(), readonly);
}
}
}
else if (entry.is_regular_file())
{
u64 filesize = entry.file_size();
auto lastmodified = entry.last_write_time();
s64 lastmodified_raw = std::chrono::duration_cast<std::chrono::seconds>(lastmodified.time_since_epoch()).count();
bool import = false;
if (FileIndex.count(innerpath) < 1)
{
import = true;
}
else
{
FileIndexEntry& chk = FileIndex[innerpath];
if (chk.Size != filesize) import = true;
if (chk.LastModified != lastmodified_raw) import = true;
}
if (import)
{
FileIndexEntry ientry;
ientry.Path = innerpath;
ientry.IsReadOnly = readonly;
ientry.Size = filesize;
ientry.LastModified = lastmodified_raw;
innerpath = "0:/" + innerpath;
if (ImportFile(innerpath, fullpath))
{
FF_FILINFO finfo;
f_stat(innerpath.c_str(), &finfo);
ientry.LastModifiedInternal = (finfo.fdate << 16) | finfo.ftime;
FileIndex[ientry.Path] = ientry;
printf("IMPORTING FILE %s (FROM %s), %d\n", innerpath.c_str(), fullpath.c_str(), readonly);
}
}
}
f_chmod(innerpath.c_str(), readonly?AM_RDO:0, AM_RDO);
}
SaveIndex();
return false;
}
return false;
}
bool FATStorage::Build(const char* sourcedir, u64 size, const char* filename)
{
FilePath = filename;
FileSize = size;
FF_File = Platform::OpenLocalFile(filename, "r+b");
if (!FF_File)
{
FF_File = Platform::OpenLocalFile(filename, "w+b");
if (!FF_File)
return false;
}
IndexPath = FilePath + ".idx";
LoadIndex();
FF_FileSize = size;
ff_disk_open(FF_ReadStorage, FF_WriteStorage, (LBA_t)(size>>9));
FRESULT res;
FATFS fs;
res = f_mount(&fs, "0:", 1);
if (res != FR_OK)
{
// TODO: determine proper FAT type!
// for example: libfat tries to determine the FAT type from the number of clusters
// which doesn't match the way fatfs handles autodetection
//
// partition->dataStart = partition->rootDirStart + (( u8array_to_u16(sectorBuffer, BPB_rootEntries) * DIR_ENTRY_DATA_SIZE) / partition->bytesPerSector);
// uint32_t clusterCount = (partition->numberOfSectors - (uint32_t)(partition->dataStart - startSector)) / partition->sectorsPerCluster;
// FAT12: max cluster count 4085
// FAT16; max cluster count: 65525
//
FF_MKFS_PARM fsopt;
fsopt.fmt = FM_FAT;// | FM_FAT32;
fsopt.au_size = 0;
fsopt.align = 1;
fsopt.n_fat = 1;
fsopt.n_root = 512;
BYTE workbuf[FF_MAX_SS];
res = f_mkfs("0:", &fsopt, workbuf, sizeof(workbuf));
if (res == FR_OK)
res = f_mount(&fs, "0:", 1);
}
if (res == FR_OK)
BuildSubdirectory(sourcedir, "", 0);
f_unmount("0:");
ff_disk_close();
fclose(FF_File);
FF_File = nullptr;
return true;
}
bool FATStorage::Save(std::string sourcedir)
{
FF_File = Platform::OpenLocalFile(FilePath.c_str(), "r+b");
if (!FF_File)
{
return false;
}
FF_FileSize = FileSize;
ff_disk_open(FF_ReadStorage, FF_WriteStorage, (LBA_t)(FileSize>>9));
FRESULT res;
FATFS fs;
res = f_mount(&fs, "0:", 1);
if (res != FR_OK)
{
ff_disk_close();
fclose(FF_File);
FF_File = nullptr;
return false;
}
ExportChanges("dldi");
SaveIndex();
f_unmount("0:");
ff_disk_close();
fclose(FF_File);
FF_File = nullptr;
return true;
}