Shader Disk Cache implementation for D3D. Saves generated shaders on disk. Eliminates "freeze jerks" in D3D plugin the _second_ and later times you play something.... not much to do about the first time. The D3D shader compiler is just slow.

Also assorted cleanup around the shader code.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@4869 8ced0084-cf51-0410-be5f-012b33b47a6e

Source/Core/Common/Src/CommonPaths.h

@@ -66,6 +66,7 @@
 #define GAMECONFIG_DIR "GameConfig"
 #define MAPS_DIR "Maps"
 #define CACHE_DIR "Cache"
+#define SHADERCACHE_DIR "ShaderCache"
 #define STATESAVES_DIR "StateSaves"
 #define SCREENSHOTS_DIR "ScreenShots"
 #define DUMP_DIR "Dump"
@@ -128,6 +129,7 @@
 
 #define FULL_CONFIG_DIR		FULL_USERDATA_DIR CONFIG_DIR DIR_SEP
 #define FULL_CACHE_DIR		FULL_USERDATA_DIR CACHE_DIR DIR_SEP 
+#define FULL_SHADERCACHE_DIR FULL_USERDATA_DIR SHADERCACHE_DIR DIR_SEP 
 #define FULL_STATESAVES_DIR	FULL_USERDATA_DIR STATESAVES_DIR DIR_SEP
 #define FULL_SCREENSHOTS_DIR FULL_USERDATA_DIR SCREENSHOTS_DIR DIR_SEP
 #define FULL_FRAMES_DIR		FULL_USERDATA_DIR DUMP_DIR DIR_SEP DUMP_FRAMES_DIR 

Source/Core/Common/Src/LinearDiskCache.cpp

@@ -0,0 +1,149 @@
+// Copyright (C) 2003 Dolphin Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official SVN repository and contact information can be found at
+// http://code.google.com/p/dolphin-emu/
+
+#include "LinearDiskCache.h"
+
+static const char ID[4] = {'D', 'C', 'A', 'C'};
+const int version = 1;  // TODO: Get from SVN_REV
+
+LinearDiskCache::LinearDiskCache() 
+	: file_(NULL), num_entries_(0) {
+}
+
+void LinearDiskCache::WriteHeader() {
+	fwrite(ID, 4, 1, file_);
+	fwrite(&version, 4, 1, file_);
+}
+
+bool LinearDiskCache::ValidateHeader() {
+	char header_id[4];
+	int header_version;
+	fread(&header_id, 4, 1, file_);
+	fread(&header_version, 4, 1, file_);
+	if (memcmp(header_id, ID, 4) != 0)
+		return false;
+	if (header_version != version)
+		return false;
+	return true;
+}
+
+int LinearDiskCache::OpenAndRead(const char *filename, LinearDiskCacheReader *reader) {
+	int items_read_count = 0;
+	file_ = fopen(filename, "rb");
+	int file_size = 0;
+	if (file_) {
+		fseek(file_, 0, SEEK_END);
+		file_size = (int)ftell(file_);
+	}
+
+	bool file_corrupt = false;
+	if (file_size == 0) {
+		if (file_)
+			fclose(file_);
+		// Reopen for writing.
+		file_ = fopen(filename, "wb");
+		// Cache empty, let's initialize a header.
+		WriteHeader();
+		num_entries_ = 0;
+	} else {
+		// file_ must be != 0 here.
+		// Back to the start we go.
+		fseek(file_, 0, SEEK_SET);
+		// Check that the file is valid
+		if (!ValidateHeader()) {
+			// Not valid - delete the file and start over.
+			fclose(file_);
+			unlink(filename);
+
+			PanicAlert("LinearDiskCache file header broken.");
+
+			file_ = fopen(filename, "wb");
+			WriteHeader();
+			num_entries_ = 0;
+		} else {
+			// Valid - blow through it.
+			// We're past the header already thanks to ValidateHeader.
+			while (!feof(file_)) {
+				int key_size, value_size;
+				int key_size_size = fread(&key_size, 1, sizeof(key_size), file_);
+				int value_size_size = fread(&value_size, 1, sizeof(value_size), file_);
+				if (key_size_size == 0 && value_size_size == 0) {
+					// I guess feof isn't doing it's job - we're at the end.
+					break;
+				}
+				if (key_size <= 0 || value_size < 0 || key_size_size != 4 || value_size_size != 4) {
+					PanicAlert("Disk cache file %s corrupted/truncated! ks: %i vs %i kss %i vss %i", filename,
+						key_size, value_size, key_size_size, value_size_size);
+					file_corrupt = true;
+					break;
+				}
+				u8 *key = new u8[key_size];
+				u8 *value = new u8[value_size];
+				int actual_key_size = (int)fread(key, 1, key_size, file_);
+				int actual_value_size = (int)fread(value, 1, value_size, file_);
+				if (actual_key_size != key_size || actual_value_size != value_size) {
+					PanicAlert("Disk cache file %s corrupted/truncated! ks: %i  actual ks: %i   vs: %i  actual vs: %i", filename,
+						key_size, actual_key_size, value_size, actual_value_size);
+					file_corrupt = true;
+				} else {
+					reader->Read(key, key_size, value, value_size);
+					items_read_count++;
+				}
+				delete [] key;
+				delete [] value;
+			}
+			fclose(file_);
+			// Done reading.
+
+			// Reopen file for append.
+			// At this point, ftell() will be at the end of the file,
+			// which happens to be exactly what we want.
+			file_ = fopen(filename, "ab");
+			fseek(file_, 0, SEEK_END);
+		}
+	}
+
+	if (file_corrupt) {
+		// Restore sanity, start over.
+		fclose(file_);
+		unlink(filename);
+
+		file_ = fopen(filename, "wb+");
+		WriteHeader();
+	}
+
+	return items_read_count;
+}
+
+void LinearDiskCache::Append(
+	const u8 *key, int key_size, const u8 *value, int value_size) {
+	// Should do a check that we don't already have "key"?
+	fwrite(&key_size, 1, sizeof(key_size), file_);
+	fwrite(&value_size, 1, sizeof(value_size), file_);
+	fwrite(key, 1, key_size, file_);
+	fwrite(value, 1, value_size, file_);
+}
+
+void LinearDiskCache::Sync() {
+	fflush(file_);
+}
+
+void LinearDiskCache::Close() {
+	fclose(file_);
+	file_ = 0;
+	num_entries_ = 0;
+}

Source/Core/Common/Src/LinearDiskCache.h

@@ -0,0 +1,67 @@
+// Copyright (C) 2003 Dolphin Project.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, version 2.0.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License 2.0 for more details.
+
+// A copy of the GPL 2.0 should have been included with the program.
+// If not, see http://www.gnu.org/licenses/
+
+// Official SVN repository and contact information can be found at
+// http://code.google.com/p/dolphin-emu/
+
+#ifndef _LINEAR_DISKCACHE
+#define _LINEAR_DISKCACHE
+
+#include "Common.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// On disk format:
+// uint32 'DCAC'
+// uint32 version;  // svn_rev
+// uint32 key_length;
+// uint32 value_length;
+// ....   key;
+// ....   value;
+
+class LinearDiskCacheReader {
+public:
+	virtual void Read(const u8 *key, int key_size, const u8 *value, int value_size) = 0;
+};
+
+// Dead simple unsorted key-value store with append functionality.
+// No random read functionality, all reading is done in OpenAndRead.
+// Keys and values can contain any characters, including \0.
+//
+// Suitable for caching generated shader bytecode between executions.
+// Not tuned for extreme performance but should be reasonably fast.
+// Does not support keys or values larger than 2GB, which should be reasonable.
+// Keys must have non-zero length; values can have zero length.
+class LinearDiskCache {
+public:
+	LinearDiskCache();
+
+	// Returns the number of items read from the cache.
+	int OpenAndRead(const char *filename, LinearDiskCacheReader *reader);
+	void Close();
+	void Sync();
+
+	// Appends a key-value pair to the store.
+	void Append(const u8 *key, int key_size, const u8 *value, int value_size);
+
+private:
+	void WriteHeader();
+	bool ValidateHeader();
+
+	FILE *file_;
+	int num_entries_;
+};
+
+#endif  // _LINEAR_DISKCACHE