Large blob code cleanup. Compressed ISO ("gcz") support reactivated. Beginnings of raw drive reading code. Deprecate file mapping in an ugly way.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@566 8ced0084-cf51-0410-be5f-012b33b47a6e
This commit is contained in:
hrydgard
2008-09-17 23:25:35 +00:00
parent 2cd7acfaee
commit cb90e61368
31 changed files with 1209 additions and 844 deletions

View File

@ -1,567 +1,110 @@
#ifdef WIN32
#include <io.h>
#include <windows.h>
#else
#include <unistd.h>
#endif
// Copyright (C) 2003-2008 Dolphin Project.
#include "stdafx.h"
#include "../../../../Externals/zlib/zlib.h"
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#include "Common.h"
#include "FileUtil.h"
#include "Blob.h"
#include "CompressedBlob.h"
#include "FileBlob.h"
#include "DriveBlob.h"
#include "MappedFile.h"
namespace DiscIO
{
const u32 kBlobCookie = 0xB10BC001;
// A blob file structure:
// BlobHeader
// u64 offsetsToBlocks[n], top bit specifies whether the block is compressed, or not.
// compressed data
// Provides caching and split-operation-to-block-operations facilities.
// Used for compressed blob reading and direct drive reading.
// Blocks that won't compress to less than 97% of the original size are stored as-is.
struct BlobHeader // 32 bytes
void SectorReader::SetSectorSize(int blocksize)
{
u32 magic_cookie; //0xB10BB10B
u32 sub_type; // gc image, whatever
u64 compressed_data_size;
u64 data_size;
u32 block_size;
u32 num_blocks;
};
#ifdef _WIN32
class PlainFileReader
: public IBlobReader
{
HANDLE hFile;
s64 size;
private:
PlainFileReader(HANDLE hFile_)
{
hFile = hFile_;
DWORD size_low, size_high;
size_low = GetFileSize(hFile, &size_high);
size = ((u64)size_low) | ((u64)size_high << 32);
}
public:
static PlainFileReader* Create(const char* filename)
{
HANDLE hFile = CreateFile(
filename, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, NULL
);
if (hFile != INVALID_HANDLE_VALUE)
{
return new PlainFileReader(hFile);
}
return 0;
}
~PlainFileReader()
{
CloseHandle(hFile);
}
u64 GetDataSize() const {return(size);}
u64 GetRawSize() const {return(size);}
bool Read(u64 offset, u64 size, u8* out_ptr)
{
LONG offset_high = (LONG)(offset >> 32);
SetFilePointer(hFile, (DWORD)(offset & 0xFFFFFFFF), &offset_high, FILE_BEGIN);
if (size >= 0x100000000ULL)
{
return(false); // WTF, does windows really have this limitation?
}
DWORD unused;
if (!ReadFile(hFile, out_ptr, DWORD(size & 0xFFFFFFFF), &unused, NULL))
{
return(false);
}
else
{
return(true);
}
}
};
#else // linux, 64-bit. We do not yet care about linux32
// Not optimal - will keep recreating mappings.
class PlainFileReader : public IBlobReader
{
FILE* file_;
s64 size;
private:
PlainFileReader(FILE* file__)
{
file_ = file__;
#if 0
fseek64(file_, 0, SEEK_END);
#else
fseek(file_, 0, SEEK_END); // I don't have fseek64 with gcc 4.3
#endif
size = ftell(file_);
fseek(file_, 0, SEEK_SET);
}
public:
static PlainFileReader* Create(const char* filename)
{
FILE* file_ = fopen(filename, "rb");
if (file_)
{
return new PlainFileReader(file_);
}
return 0;
}
~PlainFileReader()
{
fclose(file_);
}
u64 GetDataSize() const
{
return(size);
}
u64 GetRawSize() const
{
return(size);
}
bool Read(u64 offset, u64 nbytes, u8* out_ptr)
{
fseek(file_, offset, SEEK_SET);
fread(out_ptr, nbytes, 1, file_);
return(true);
}
};
#endif
class CompressedBlobReader
: public IBlobReader
{
enum { CACHE_SIZE = 32 };
BlobHeader header;
u64* block_pointers;
int data_offset;
u8* cache[CACHE_SIZE];
u64 cache_tags[CACHE_SIZE];
int cache_age[CACHE_SIZE];
u64 counter;
Common::IMappedFile* mapped_file;
private:
CompressedBlobReader(Common::IMappedFile* mapped_file_)
{
mapped_file = mapped_file_;
counter = 0;
u8* start = mapped_file->Lock(0, sizeof(BlobHeader));
memcpy(&header, start, sizeof(BlobHeader));
mapped_file->Unlock(start);
block_pointers = (u64*)mapped_file->Lock(sizeof(BlobHeader), sizeof(u64) * header.num_blocks);
data_offset = sizeof(BlobHeader) + sizeof(u64) * header.num_blocks;
for (int i = 0; i < CACHE_SIZE; i++)
{
cache[i] = new u8[header.block_size];
cache_tags[i] = (u64)(s64) - 1;
}
}
public:
static CompressedBlobReader* Create(const char* filename)
{
Common::IMappedFile* mapped_file =
Common::IMappedFile::CreateMappedFile();
if (mapped_file)
{
bool ok = mapped_file->Open(filename);
if (ok)
{
return new CompressedBlobReader(mapped_file);
}
else
{
delete mapped_file;
}
}
return 0;
}
~CompressedBlobReader()
{
for (int i = 0; i < CACHE_SIZE; i++)
{
delete[] cache[i];
}
mapped_file->Unlock((u8*)block_pointers);
mapped_file->Close();
delete mapped_file;
}
const BlobHeader& GetHeader() const
{
return(header);
}
u64 GetDataSize() const
{
return(header.data_size);
}
u64 GetRawSize() const
{
return(mapped_file->GetSize());
}
u64 GetBlockCompressedSize(u64 block_num) const
{
u64 start = block_pointers[block_num];
if (block_num != header.num_blocks - 1)
{
return(block_pointers[block_num + 1] - start);
}
else
{
return(header.compressed_data_size - start);
}
}
// IMPORTANT: Calling this function invalidates all earlier pointers gotten from this function.
u8* GetBlock(u64 block_num)
{
if (cache_tags[0] != block_num)
{
cache_tags[0] = block_num;
//PanicAlert("here2");
// let's begin with a super naive implementation.
// let's just use the start of the cache for now.
bool uncompressed = false;
u32 comp_block_size = (u32)GetBlockCompressedSize(block_num);
u64 offset = block_pointers[block_num] + data_offset;
if (offset & (1ULL << 63))
{
if (comp_block_size != header.block_size)
{
PanicAlert("Uncompressed block with wrong size");
}
uncompressed = true;
offset &= ~(1ULL << 63);
}
u8* source = mapped_file->Lock(offset, comp_block_size + 64*1024);
u8* dest = cache[0];
if (uncompressed)
{
memcpy(dest, source, comp_block_size);
}
else
{
z_stream z;
memset(&z, 0, sizeof(z));
z.next_in = source;
z.avail_in = comp_block_size;
if (z.avail_in > header.block_size)
{
PanicAlert("We have a problem");
}
z.next_out = dest;
z.avail_out = header.block_size;
inflateInit(&z);
int status = inflate(&z, Z_FULL_FLUSH);
u32 uncomp_size = header.block_size - z.avail_out;
if (status != Z_STREAM_END)
{
// this seem to fire wrongly from time to time
// to be sure, don't use compressed isos :P
//PanicAlert("Failure reading block %i", block_num);
}
if (uncomp_size != header.block_size)
{
PanicAlert("Wrong block size");
}
inflateEnd(&z);
}
mapped_file->Unlock(source);
}
//PanicAlert("here3");
return(cache[0]);
}
bool Read(u64 offset, u64 size, u8* out_ptr)
{
u64 startingBlock = offset / header.block_size;
u64 remain = size;
int positionInBlock = (int)(offset % header.block_size);
u64 block = startingBlock;
while (remain > 0)
{
u8* data = GetBlock(block);
if (!data)
{
return(false);
}
u32 toCopy = header.block_size - positionInBlock;
if (toCopy >= remain)
{
// yay, we are done!
memcpy(out_ptr, data + positionInBlock, (size_t)remain);
return(true);
}
else
{
memcpy(out_ptr, data + positionInBlock, toCopy);
out_ptr += toCopy;
remain -= toCopy;
positionInBlock = 0;
block++;
}
}
PanicAlert("here4");
return(true);
}
};
bool CompressFileToBlob(const char* infile, const char* outfile, u32 sub_type, int block_size,
CompressCB callback, void* arg)
{
//Common::IMappedFile *in = Common::IMappedFile::CreateMappedFile();
if (IsCompressedBlob(infile))
for (int i = 0; i < CACHE_SIZE; i++)
{
PanicAlert("%s is already compressed!", infile);
return(false);
cache[i] = new u8[blocksize];
cache_tags[i] = (u64)(s64) - 1;
}
m_blocksize = blocksize;
}
FILE* inf = fopen(infile, "rb");
SectorReader::~SectorReader() {
for (int i = 0; i < CACHE_SIZE; i++)
delete[] cache[i];
}
if (!inf)
const u8 *SectorReader::GetBlockData(u64 block_num)
{
if (cache_tags[0] == block_num)
{
return(false);
return cache[0];
}
FILE* f = fopen(outfile, "wb");
if (!f)
else
{
return(false);
GetBlock(block_num, cache[0]);
cache_tags[0] = block_num;
return cache[0];
}
}
callback("Files opened, ready to compress.", 0, arg);
bool SectorReader::Read(u64 offset, u64 size, u8* out_ptr)
{
u64 startingBlock = offset / m_blocksize;
u64 remain = size;
fseek(inf, 0, SEEK_END);
int insize = ftell(inf);
fseek(inf, 0, SEEK_SET);
BlobHeader header;
header.magic_cookie = kBlobCookie;
header.sub_type = sub_type;
header.block_size = block_size;
header.data_size = insize;
int positionInBlock = (int)(offset % m_blocksize);
u64 block = startingBlock;
// round upwards!
header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size);
u64* offsets = new u64[header.num_blocks];
u8* out_buf = new u8[block_size];
u8* in_buf = new u8[block_size];
// seek past the header (we will write it at the end)
fseek(f, sizeof(BlobHeader), SEEK_CUR);
// seek past the offset table (we will write it at the end)
fseek(f, sizeof(u64) * header.num_blocks, SEEK_CUR);
// Now we are ready to write compressed data!
u64 position = 0;
int num_compressed = 0;
int num_stored = 0;
for (u32 i = 0; i < header.num_blocks; i++)
while (remain > 0)
{
if (i % (header.num_blocks / 1000) == 0)
const u8* data = GetBlockData(block);
if (!data)
return false;
u32 toCopy = m_blocksize - positionInBlock;
if (toCopy >= remain)
{
u64 inpos = ftell(inf);
int ratio = 0;
if (inpos != 0)
{
ratio = (int)(100 * position / inpos);
}
char temp[512];
sprintf(temp, "%i of %i blocks. compression ratio %i%%", i, header.num_blocks, ratio);
callback(temp, (float)i / (float)header.num_blocks, arg);
}
offsets[i] = position;
// u64 start = i * header.block_size;
// u64 size = header.block_size;
memset(in_buf, 0, header.block_size);
fread(in_buf, header.block_size, 1, inf);
z_stream z;
memset(&z, 0, sizeof(z));
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
z.next_in = in_buf;
z.avail_in = header.block_size;
z.next_out = out_buf;
z.avail_out = block_size;
int retval = deflateInit(&z, 9);
if (retval != Z_OK)
{
PanicAlert("Deflate failed");
goto cleanup;
}
int status = deflate(&z, Z_FINISH);
int comp_size = block_size - z.avail_out;
if ((status != Z_STREAM_END) || (z.avail_out < 10))
{
//PanicAlert("%i %i Store %i", i*block_size, position, comp_size);
// let's store uncompressed
offsets[i] |= 0x8000000000000000ULL;
fwrite(in_buf, block_size, 1, f);
position += block_size;
num_stored++;
// yay, we are done!
memcpy(out_ptr, data + positionInBlock, (size_t)remain);
return true;
}
else
{
// let's store compressed
//PanicAlert("Comp %i to %i", block_size, comp_size);
fwrite(out_buf, comp_size, 1, f);
position += comp_size;
num_compressed++;
memcpy(out_ptr, data + positionInBlock, toCopy);
out_ptr += toCopy;
remain -= toCopy;
positionInBlock = 0;
block++;
}
deflateEnd(&z);
}
header.compressed_data_size = position;
// Okay, go back and fill in headers
fseek(f, 0, SEEK_SET);
fwrite(&header, sizeof(header), 1, f);
fwrite(offsets, sizeof(u64), header.num_blocks, f);
cleanup:
// Cleanup
delete[] in_buf;
delete[] out_buf;
delete[] offsets;
fclose(f);
fclose(inf);
callback("Done.", 1.0f, arg);
return(true);
}
bool DecompressBlobToFile(const char* infile, const char* outfile,
CompressCB callback, void* arg)
{
if (!IsCompressedBlob(infile))
{
PanicAlert("File not compressed");
return(false);
}
CompressedBlobReader* reader = CompressedBlobReader::Create(infile);
if (!reader) return false;
FILE* f = fopen(outfile, "wb");
const BlobHeader& header = reader->GetHeader();
u8* buffer = new u8[header.block_size];
for (u64 i = 0; i < header.num_blocks; i++)
{
if (i % (header.num_blocks / 100) == 0)
{
callback("Unpacking", (float)i / (float)header.num_blocks, arg);
}
reader->Read(i * header.block_size, header.block_size, buffer);
fwrite(buffer, header.block_size, 1, f);
}
delete reader;
delete[] buffer;
#ifdef _WIN32
// TODO(ector): _chsize sucks, not 64-bit safe
// F|RES: changed to _chsize_s. i think it is 64-bit safe
_chsize_s(_fileno(f), (long)header.data_size);
#else
ftruncate(fileno(f), header.data_size);
#endif
fclose(f);
return(true);
}
bool IsCompressedBlob(const char* filename)
{
FILE* f = fopen(filename, "rb");
if (!f)
{
return(0);
}
BlobHeader header;
fread(&header, sizeof(header), 1, f);
fclose(f);
return(header.magic_cookie == kBlobCookie);
return true;
}
IBlobReader* CreateBlobReader(const char* filename)
{
return IsCompressedBlob(filename)
? static_cast<IBlobReader*>(CompressedBlobReader::Create(filename))
: static_cast<IBlobReader*>(PlainFileReader::Create(filename));
//if (strlen(filename) < 4 && filename[1] == ':') // Drive, for sure.
// return DriveReader::Create(filename);
if (!File::Exists(filename))
return 0;
if (IsCompressedBlob(filename))
return CompressedBlobReader::Create(filename);
// Still here? Assume plain file.
return PlainFileReader::Create(filename);
}
} // namespace
} // namespace

View File

@ -1,54 +1,73 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#ifndef _BLOB_H
#define _BLOB_H
/*
Code not big-endian safe.
// BLOB
BLOB
// Blobs in Dolphin are read only Binary Large OBjects. For example, a typical DVD image.
// Often, you may want to store these things in a highly compressed format, but still
// allow random access. Or you may store them on an odd device, like raw on a DVD.
Blobs are Binary Large OBjects. For example, a typical DVD image.
Often, you may want to store these things in a highly compressed format, but still
allow random access.
Always read your BLOBs using an interface returned by CreateBlobReader(). It will
detect whether the file is a compressed blob, or just a big hunk of data, and
automatically do the right thing.
To create new BLOBs, use CompressFileToBlob.
Right now caching of decompressed chunks doesn't work, so it's a bit slow.
*/
// Always read your BLOBs using an interface returned by CreateBlobReader(). It will
// detect whether the file is a compressed blob, or just a big hunk of data, and
// automatically do the right thing.
#include "Common.h"
namespace DiscIO
{
class IBlobReader
{
public:
public:
virtual ~IBlobReader() {}
virtual ~IBlobReader() {}
virtual u64 GetRawSize() const = 0;
virtual u64 GetDataSize() const = 0;
virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0;
virtual u64 GetRawSize() const = 0;
virtual u64 GetDataSize() const = 0;
virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0;
protected:
IBlobReader() {}
private:
IBlobReader(const IBlobReader& /*other*/) {}
protected:
IBlobReader() {}
};
IBlobReader* CreateBlobReader(const char* filename);
bool IsCompressedBlob(const char* filename);
// Provides caching and split-operation-to-block-operations facilities.
// Used for compressed blob reading and direct drive reading.
class SectorReader : public IBlobReader
{
private:
virtual void GetBlock(u64 block_num, u8 *out) = 0;
enum { CACHE_SIZE = 32 };
int m_blocksize;
u8* cache[CACHE_SIZE];
u64 cache_tags[CACHE_SIZE];
int cache_age[CACHE_SIZE];
protected:
void SetSectorSize(int blocksize);
public:
~SectorReader();
const u8 *GetBlockData(u64 block_num);
bool Read(u64 offset, u64 size, u8* out_ptr);
};
// Factory function - examines the path to choose the right type of IBlobReader, and returns one.
IBlobReader* CreateBlobReader(const char* filename);
typedef void (*CompressCB)(const char* text, float percent, void* arg);
@ -56,7 +75,8 @@ bool CompressFileToBlob(const char* infile, const char* outfile, u32 sub_type =
CompressCB callback = 0, void* arg = 0);
bool DecompressBlobToFile(const char* infile, const char* outfile,
CompressCB callback = 0, void* arg = 0);
}
} // namespace
#endif

View File

@ -0,0 +1,339 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#include "stdafx.h"
#ifdef _WIN32
#include <io.h>
#include <windows.h>
#else
#include <unistd.h>
#endif
#include "Common.h"
#include "CompressedBlob.h"
#include "Hash.h"
#ifdef _WIN32
#include "../../../../Externals/zlib/zlib.h"
#else
// TODO: Include generic zlib.h
#include "../../../../Externals/zlib/zlib.h"
#endif
#ifdef _WIN32
#define fseek _fseeki64
#endif
namespace DiscIO
{
CompressedBlobReader::CompressedBlobReader(const char *filename)
{
file = fopen(filename, "rb");
fseek(file, 0, SEEK_END);
file_size = ftell(file);
fseek(file, 0, SEEK_SET);
fread(&header, sizeof(CompressedBlobHeader), 1, file);
SetSectorSize(header.block_size);
// cache block pointers and hashes
block_pointers = new u64[header.num_blocks];
fread(block_pointers, sizeof(u64), header.num_blocks, file);
hashes = new u32[header.num_blocks];
fread(hashes, sizeof(u32), header.num_blocks, file);
data_offset = (sizeof(CompressedBlobHeader))
+ (sizeof(u64)) * header.num_blocks // skip block pointers
+ (sizeof(u32)) * header.num_blocks; // skip hashes
// A compressed block is never ever longer than a decompressed block, so just header.block_size should be fine.
// I still add some safety margin.
zlib_buffer_size = header.block_size + 64;
zlib_buffer = new u8[zlib_buffer_size];
memset(zlib_buffer, 0, zlib_buffer_size);
}
CompressedBlobReader* CompressedBlobReader::Create(const char* filename)
{
if (IsCompressedBlob(filename))
return new CompressedBlobReader(filename);
else
return 0;
}
CompressedBlobReader::~CompressedBlobReader()
{
delete [] zlib_buffer;
delete [] block_pointers;
delete [] hashes;
fclose(file);
file = 0;
}
// IMPORTANT: Calling this function invalidates all earlier pointers gotten from this function.
u64 CompressedBlobReader::GetBlockCompressedSize(u64 block_num) const
{
u64 start = block_pointers[block_num];
if (block_num < header.num_blocks - 1)
return block_pointers[block_num + 1] - start;
else if (block_num == header.num_blocks - 1)
return header.compressed_data_size - start;
else
PanicAlert("GetBlockCompressedSize - illegal block number %i", (int)block_num);
return 0;
}
void CompressedBlobReader::GetBlock(u64 block_num, u8 *out_ptr)
{
bool uncompressed = false;
u32 comp_block_size = (u32)GetBlockCompressedSize(block_num);
u64 offset = block_pointers[block_num] + data_offset;
if (offset & (1ULL << 63))
{
if (comp_block_size != header.block_size)
PanicAlert("Uncompressed block with wrong size");
uncompressed = true;
offset &= ~(1ULL << 63);
}
// clear unused part of zlib buffer. maybe this can be deleted when it works fully.
memset(zlib_buffer + comp_block_size, 0, zlib_buffer_size - comp_block_size);
fseek(file, offset, SEEK_SET);
fread(zlib_buffer, 1, comp_block_size, file);
u8* source = zlib_buffer;
u8* dest = out_ptr;
// First, check hash.
u32 block_hash = HashAdler32(source, comp_block_size);
if (block_hash != hashes[block_num])
PanicAlert("Hash of block %i is %08x instead of %08x. Your ISO is corrupt.",
block_num, block_hash, hashes[block_num]);
if (uncompressed)
{
memcpy(dest, source, comp_block_size);
}
else
{
z_stream z;
memset(&z, 0, sizeof(z));
z.next_in = source;
z.avail_in = comp_block_size;
if (z.avail_in > header.block_size)
{
PanicAlert("We have a problem");
}
z.next_out = dest;
z.avail_out = header.block_size;
inflateInit(&z);
int status = inflate(&z, Z_FULL_FLUSH);
u32 uncomp_size = header.block_size - z.avail_out;
if (status != Z_STREAM_END)
{
// this seem to fire wrongly from time to time
// to be sure, don't use compressed isos :P
PanicAlert("Failure reading block %i - out of data and not at end.", block_num);
}
inflateEnd(&z);
if (uncomp_size != header.block_size)
PanicAlert("Wrong block size");
}
}
bool CompressFileToBlob(const char* infile, const char* outfile, u32 sub_type,
int block_size, CompressCB callback, void* arg)
{
if (IsCompressedBlob(infile))
{
PanicAlert("%s is already compressed! Cannot compress it further.", infile);
return false;
}
FILE* inf = fopen(infile, "rb");
if (!inf)
return false;
FILE* f = fopen(outfile, "wb");
if (!f)
return false;
callback("Files opened, ready to compress.", 0, arg);
fseek(inf, 0, SEEK_END);
int insize = ftell(inf);
fseek(inf, 0, SEEK_SET);
CompressedBlobHeader header;
header.magic_cookie = kBlobCookie;
header.sub_type = sub_type;
header.block_size = block_size;
header.data_size = insize;
// round upwards!
header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size);
u64* offsets = new u64[header.num_blocks];
u32* hashes = new u32[header.num_blocks];
u8* out_buf = new u8[block_size];
u8* in_buf = new u8[block_size];
// seek past the header (we will write it at the end)
fseek(f, sizeof(CompressedBlobHeader), SEEK_CUR);
// seek past the offset and hash tables (we will write them at the end)
fseek(f, (sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR);
// Now we are ready to write compressed data!
u64 position = 0;
int num_compressed = 0;
int num_stored = 0;
for (u32 i = 0; i < header.num_blocks; i++)
{
if (i % (header.num_blocks / 1000) == 0)
{
u64 inpos = ftell(inf);
int ratio = 0;
if (inpos != 0)
ratio = (int)(100 * position / inpos);
char temp[512];
sprintf(temp, "%i of %i blocks. compression ratio %i%%", i, header.num_blocks, ratio);
callback(temp, (float)i / (float)header.num_blocks, arg);
}
offsets[i] = position;
// u64 start = i * header.block_size;
// u64 size = header.block_size;
memset(in_buf, 0, header.block_size);
fread(in_buf, header.block_size, 1, inf);
z_stream z;
memset(&z, 0, sizeof(z));
z.zalloc = Z_NULL;
z.zfree = Z_NULL;
z.opaque = Z_NULL;
z.next_in = in_buf;
z.avail_in = header.block_size;
z.next_out = out_buf;
z.avail_out = block_size;
int retval = deflateInit(&z, 9);
if (retval != Z_OK)
{
PanicAlert("Deflate failed");
goto cleanup;
}
int status = deflate(&z, Z_FINISH);
int comp_size = block_size - z.avail_out;
if ((status != Z_STREAM_END) || (z.avail_out < 10))
{
//PanicAlert("%i %i Store %i", i*block_size, position, comp_size);
// let's store uncompressed
offsets[i] |= 0x8000000000000000ULL;
fwrite(in_buf, block_size, 1, f);
hashes[i] = HashAdler32(in_buf, block_size);
position += block_size;
num_stored++;
}
else
{
// let's store compressed
//PanicAlert("Comp %i to %i", block_size, comp_size);
fwrite(out_buf, comp_size, 1, f);
hashes[i] = HashAdler32(out_buf, comp_size);
position += comp_size;
num_compressed++;
}
deflateEnd(&z);
}
header.compressed_data_size = position;
// Okay, go back and fill in headers
fseek(f, 0, SEEK_SET);
fwrite(&header, sizeof(header), 1, f);
fwrite(offsets, sizeof(u64), header.num_blocks, f);
fwrite(hashes, sizeof(u32), header.num_blocks, f);
cleanup:
// Cleanup
delete[] in_buf;
delete[] out_buf;
delete[] offsets;
fclose(f);
fclose(inf);
callback("Done compressing disc image.", 1.0f, arg);
return true;
}
bool DecompressBlobToFile(const char* infile, const char* outfile, CompressCB callback, void* arg)
{
if (!IsCompressedBlob(infile))
{
PanicAlert("File not compressed");
return false;
}
CompressedBlobReader* reader = CompressedBlobReader::Create(infile);
if (!reader) return false;
FILE* f = fopen(outfile, "wb");
const CompressedBlobHeader &header = reader->GetHeader();
u8* buffer = new u8[header.block_size];
for (u64 i = 0; i < header.num_blocks; i++)
{
if (i % (header.num_blocks / 100) == 0)
{
callback("Unpacking", (float)i / (float)header.num_blocks, arg);
}
reader->Read(i * header.block_size, header.block_size, buffer);
fwrite(buffer, header.block_size, 1, f);
}
delete reader;
delete[] buffer;
#ifdef _WIN32
// ector: _chsize sucks, not 64-bit safe
// F|RES: changed to _chsize_s. i think it is 64-bit safe
_chsize_s(_fileno(f), (long)header.data_size);
#else
ftruncate(fileno(f), header.data_size);
#endif
fclose(f);
return true;
}
bool IsCompressedBlob(const char* filename)
{
FILE* f = fopen(filename, "rb");
if (!f)
return false;
CompressedBlobHeader header;
fread(&header, sizeof(header), 1, f);
fclose(f);
return header.magic_cookie == kBlobCookie;
}
} // namespace

View File

@ -0,0 +1,83 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
// WARNING Code not big-endian safe.
// To create new compressed BLOBs, use CompressFileToBlob.
// File format
// * Header
// * [Block Pointers interleaved with block hashes (hash of decompressed data)]
// * [Data]
#ifndef COMPRESSED_BLOB_H_
#define COMPRESSED_BLOB_H_
#include <stdio.h>
#include "Blob.h"
namespace DiscIO
{
bool IsCompressedBlob(const char* filename);
const u32 kBlobCookie = 0xB10BC001;
// A blob file structure:
// BlobHeader
// u64 offsetsToBlocks[n], top bit specifies whether the block is compressed, or not.
// compressed data
// Blocks that won't compress to less than 97% of the original size are stored as-is.
struct CompressedBlobHeader // 32 bytes
{
u32 magic_cookie; //0xB10BB10B
u32 sub_type; // gc image, whatever
u64 compressed_data_size;
u64 data_size;
u32 block_size;
u32 num_blocks;
};
class CompressedBlobReader : public SectorReader
{
private:
CompressedBlobHeader header;
u64 *block_pointers;
u32 *hashes;
int data_offset;
FILE *file;
u64 file_size;
u8 *zlib_buffer;
int zlib_buffer_size;
CompressedBlobReader(const char *filename);
public:
static CompressedBlobReader* Create(const char *filename);
~CompressedBlobReader();
const CompressedBlobHeader &GetHeader() const { return header; }
u64 GetDataSize() const { return header.data_size; }
u64 GetRawSize() const { return file_size; }
u64 GetBlockCompressedSize(u64 block_num) const;
void GetBlock(u64 block_num, u8 *out_ptr);
};
} // namespace
#endif // COMPRESSED_BLOB_H_

View File

@ -0,0 +1,25 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#include "stdafx.h"
#include "DriveBlob.h"
namespace DiscIO
{
} // namespace

View File

@ -0,0 +1,60 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#ifndef _DRIVE_BLOB_H
#define _DRIVE_BLOB_H
#include "Blob.h"
#ifdef _WIN32
#include <windows.h>
#endif
namespace DiscIO
{
#ifdef _WIN32
class DriveReader : public SectorReader
{
HANDLE hDisc;
private:
DriveReader(const char *drive) {
/*
char path[MAX_PATH];
strncpy(path, drive, 3);
path[2] = 0;
sprintf(path, "\\\\.\\%s", drive);
hDisc = CreateFile(path, 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
SetSectorSize(2048);
*/
}
public:
static DriveReader *Create(const char *drive) {
return NULL;// new DriveReader(drive);
}
};
#endif
} // namespace
#endif // _DRIVE_BLOB_H

View File

@ -0,0 +1,121 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#include "stdafx.h"
#include "Blob.h"
#include "FileBlob.h"
#ifdef _WIN32
#include <windows.h>
namespace DiscIO
{
PlainFileReader::PlainFileReader(HANDLE hFile_)
{
hFile = hFile_;
DWORD size_low, size_high;
size_low = GetFileSize(hFile, &size_high);
size = ((u64)size_low) | ((u64)size_high << 32);
}
PlainFileReader* PlainFileReader::Create(const char* filename)
{
HANDLE hFile = CreateFile(
filename, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, NULL);
if (hFile != INVALID_HANDLE_VALUE)
return new PlainFileReader(hFile);
else
return 0;
}
PlainFileReader::~PlainFileReader()
{
CloseHandle(hFile);
}
bool PlainFileReader::Read(u64 offset, u64 size, u8* out_ptr)
{
LONG offset_high = (LONG)(offset >> 32);
SetFilePointer(hFile, (DWORD)(offset & 0xFFFFFFFF), &offset_high, FILE_BEGIN);
if (size >= 0x100000000ULL)
return false; // WTF, does windows really have this limitation?
DWORD unused;
if (!ReadFile(hFile, out_ptr, DWORD(size & 0xFFFFFFFF), &unused, NULL))
return false;
else
return true;
}
} // namespace
#else // linux, 64-bit. We do not yet care about linux32
namespace DiscIO
{
class PlainFileReader : public IBlobReader
{
FILE* file_;
s64 size;
private:
PlainFileReader(FILE* file__)
{
file_ = file__;
#if 0
fseek64(file_, 0, SEEK_END);
#else
fseek(file_, 0, SEEK_END); // I don't have fseek64 with gcc 4.3
#endif
size = ftell(file_);
fseek(file_, 0, SEEK_SET);
}
public:
static PlainFileReader* Create(const char* filename)
{
FILE* file_ = fopen(filename, "rb");
if (file_)
{
return new PlainFileReader(file_);
}
return 0;
}
~PlainFileReader()
{
fclose(file_);
}
u64 GetDataSize() const { return(size); }
u64 GetRawSize() const { return(size); }
bool Read(u64 offset, u64 nbytes, u8* out_ptr)
{
fseek(file_, offset, SEEK_SET);
fread(out_ptr, nbytes, 1, file_);
return true;
}
};
} // namespace
#endif

View File

@ -0,0 +1,48 @@
// Copyright (C) 2003-2008 Dolphin Project.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, version 2.0.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License 2.0 for more details.
// A copy of the GPL 2.0 should have been included with the program.
// If not, see http://www.gnu.org/licenses/
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#ifndef _FILE_BLOB_H
#define _FILE_BLOB_H
#include "Blob.h"
#ifdef _WIN32
#include <windows.h>
#endif
namespace DiscIO
{
class PlainFileReader : public IBlobReader
{
#ifdef _WIN32
HANDLE hFile;
s64 size;
PlainFileReader(HANDLE hFile_);
#endif
public:
static PlainFileReader* Create(const char* filename);
~PlainFileReader();
u64 GetDataSize() const { return size; }
u64 GetRawSize() const { return size; }
bool Read(u64 offset, u64 size, u8* out_ptr);
};
} // namespace
#endif // _FILE_BLOB_H

View File

@ -4,6 +4,9 @@ files = ["BannerLoader.cpp",
"BannerLoaderGC.cpp",
"BannerLoaderWii.cpp",
"Blob.cpp",
"CompressedBlob.cpp",
"DriveBlob.cpp",
"FileBlob.cpp",
"FileHandlerARC.cpp",
"Filesystem.cpp",
"FileSystemGCWii.cpp",

View File

@ -14,6 +14,7 @@
// Official SVN repository and contact information can be found at
// http://code.google.com/p/dolphin-emu/
#ifndef _VOLUME_H
#define _VOLUME_H

View File

@ -27,7 +27,6 @@
#include "Hash.h"
namespace DiscIO
{
enum EDiscType
@ -37,33 +36,29 @@ enum EDiscType
DISC_TYPE_WII_CONTAINER,
DISC_TYPE_GC
};
#ifndef _WIN32
struct SPartition
{
u64 Offset;
u32 Type;
}; //gcc 4.3 cries if it's local
struct SPartition
{
u64 Offset;
u32 Type;
}; //gcc 4.3 cries if it's local
#endif
class CBlobBigEndianReader
{
public:
public:
CBlobBigEndianReader(IBlobReader& _rReader) : m_rReader(_rReader) {}
CBlobBigEndianReader(IBlobReader& _rReader)
: m_rReader(_rReader)
{}
u32 Read32(u64 _Offset)
{
u32 Temp;
m_rReader.Read(_Offset, 4, (u8*)&Temp);
return(Common::swap32(Temp));
}
u32 Read32(u64 _Offset)
{
u32 Temp;
m_rReader.Read(_Offset, 4, (u8*)&Temp);
return(Common::swap32(Temp));
}
private:
IBlobReader& m_rReader;
private:
IBlobReader& m_rReader;
};
unsigned char g_MasterKey[16];
@ -72,15 +67,12 @@ bool g_MasterKeyInit = false;
IVolume* CreateVolumeFromCryptedWiiImage(IBlobReader& _rReader, u32 _VolumeType);
EDiscType GetDiscType(IBlobReader& _rReader);
IVolume* CreateVolumeFromFilename(const std::string& _rFilename)
{
IBlobReader* pReader = CreateBlobReader(_rFilename.c_str());
if (pReader == NULL)
{
return(NULL);
}
return NULL;
switch (GetDiscType(*pReader))
{
@ -104,53 +96,40 @@ IVolume* CreateVolumeFromFilename(const std::string& _rFilename)
case DISC_TYPE_UNK:
default:
delete pReader;
return(NULL);
return NULL;
}
// unreachable code
return(NULL);
return NULL;
}
bool IsVolumeWiiDisc(const IVolume& _rVolume)
{
u32 MagicWord = 0;
_rVolume.Read(0x18, 4, (u8*)&MagicWord);
if (Common::swap32(MagicWord) == 0x5D1C9EA3)
{
return(true);
}
return(false);
return (Common::swap32(MagicWord) == 0x5D1C9EA3);
}
IVolume* CreateVolumeFromCryptedWiiImage(IBlobReader& _rReader, u32 _VolumeType)
{
if (!g_MasterKeyInit)
{
FILE* pT = fopen("WII/masterkey.bin", "rb");
if (pT == NULL)
{
PanicAlert("Can't open WII/masterkey.bin");
return(NULL);
return NULL;
}
fread(g_MasterKey, 16, 1, pT);
fclose(pT);
const u32 keyhash = 0x4bc30936;
u32 hash = HashAdler32(g_MasterKey, 16);
if (hash != keyhash)
{
PanicAlert("Your Wii disc decryption key is bad.", keyhash, hash);
}
else
{
g_MasterKeyInit = true;
}
}
CBlobBigEndianReader Reader(_rReader);
@ -164,7 +143,7 @@ IVolume* CreateVolumeFromCryptedWiiImage(IBlobReader& _rReader, u32 _VolumeType)
u32 Type;
};
#endif
std::vector<SPartition>PartitionsVec;
std::vector<SPartition> PartitionsVec;
// read all partitions
for (u32 i = 0; i < numPartitions; i++)
@ -195,14 +174,13 @@ IVolume* CreateVolumeFromCryptedWiiImage(IBlobReader& _rReader, u32 _VolumeType)
u8 VolumeKey[16];
AES_cbc_encrypt(SubKey, VolumeKey, 16, &AES_KEY, IV, AES_DECRYPT);
return(new CVolumeWiiCrypted(&_rReader, rPartition.Offset + 0x20000, VolumeKey));
return new CVolumeWiiCrypted(&_rReader, rPartition.Offset + 0x20000, VolumeKey);
}
}
return(NULL);
return NULL;
}
EDiscType GetDiscType(IBlobReader& _rReader)
{
CBlobBigEndianReader Reader(_rReader);
@ -214,13 +192,9 @@ EDiscType GetDiscType(IBlobReader& _rReader)
if (MagicWord == 0x5D1C9EA3)
{
if (Reader.Read32(0x60) != 0)
{
return(DISC_TYPE_WII);
}
else
{
return(DISC_TYPE_WII_CONTAINER);
}
}
}
@ -229,12 +203,10 @@ EDiscType GetDiscType(IBlobReader& _rReader)
u32 MagicWord = Reader.Read32(0x1C);
if (MagicWord == 0xC2339F3D)
{
return(DISC_TYPE_GC);
}
}
return(DISC_TYPE_UNK);
return DISC_TYPE_UNK;
}
} // namespace
} // namespace