Add an Analytics reporting system.

Fully opt-in, reports to analytics.dolphin-emu.org over SSL. Collects system
information and settings at Dolphin start time and game start time.

UI not implemented yet, so users are required to opt in through config editing.

Source/Core/Common/Analytics.cpp

@@ -0,0 +1,231 @@
+// Copyright 2016 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#include <cmath>
+#include <cstdio>
+#include <string>
+#include <curl/curl.h>
+
+#include "Common/Analytics.h"
+#include "Common/CommonTypes.h"
+#include "Common/StringUtil.h"
+
+namespace Common
+{
+namespace
+{
+// Format version number, used as the first byte of every report sent.
+// Increment for any change to the wire format.
+constexpr u8 WIRE_FORMAT_VERSION = 0;
+
+// Identifiers for the value types supported by the analytics reporting wire
+// format.
+enum class TypeId : u8
+{
+	STRING = 0,
+	BOOL = 1,
+	UINT = 2,
+	SINT = 3,
+	FLOAT = 4,
+};
+
+void AppendBool(std::string* out, bool v)
+{
+	out->push_back(v ? '\xFF' : '\x00');
+}
+
+void AppendVarInt(std::string* out, u64 v)
+{
+	do
+	{
+		u8 current_byte = v & 0x7F;
+		v >>= 7;
+		current_byte |= (!!v) << 7;
+		out->push_back(current_byte);
+	}
+	while (v);
+}
+
+void AppendBytes(std::string* out, const u8* bytes, u32 length,
+                 bool encode_length = true)
+{
+	if (encode_length)
+	{
+		AppendVarInt(out, length);
+	}
+	out->append(reinterpret_cast<const char*>(bytes), length);
+}
+
+void AppendType(std::string* out, TypeId type)
+{
+	out->push_back(static_cast<u8>(type));
+}
+
+// Dummy write function for curl.
+size_t DummyCurlWriteFunction(char* ptr, size_t size, size_t nmemb, void* userdata)
+{
+	return size * nmemb;
+}
+
+}  // namespace
+
+AnalyticsReportBuilder::AnalyticsReportBuilder()
+{
+	m_report.push_back(WIRE_FORMAT_VERSION);
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   const std::string& v)
+{
+	AppendType(report, TypeId::STRING);
+	AppendBytes(report, reinterpret_cast<const u8*>(v.data()), static_cast<u32>(v.size()));
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   const char* v)
+{
+	AppendSerializedValue(report, std::string(v));
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   bool v)
+{
+	AppendType(report, TypeId::BOOL);
+	AppendBool(report, v);
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   u64 v)
+{
+	AppendType(report, TypeId::UINT);
+	AppendVarInt(report, v);
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   s64 v)
+{
+	AppendType(report, TypeId::SINT);
+	AppendBool(report, v >= 0);
+	AppendVarInt(report, static_cast<u64>(std::abs(v)));
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   u32 v)
+{
+	AppendSerializedValue(report, static_cast<u64>(v));
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   s32 v)
+{
+	AppendSerializedValue(report, static_cast<s64>(v));
+}
+
+void AnalyticsReportBuilder::AppendSerializedValue(std::string* report,
+                                                   float v)
+{
+	AppendType(report, TypeId::FLOAT);
+	AppendBytes(report, reinterpret_cast<u8*>(&v), sizeof (v), false);
+}
+
+AnalyticsReporter::AnalyticsReporter()
+{
+	m_reporter_thread = std::thread(&AnalyticsReporter::ThreadProc, this);
+}
+
+AnalyticsReporter::~AnalyticsReporter()
+{
+	// Set the exit request flag and wait for the thread to honor it.
+	m_reporter_stop_request.Set();
+	m_reporter_event.Set();
+	m_reporter_thread.join();
+}
+
+void AnalyticsReporter::Send(AnalyticsReportBuilder&& report)
+{
+	// Put a bound on the size of the queue to avoid uncontrolled memory growth.
+	constexpr u32 QUEUE_SIZE_LIMIT = 25;
+	if (m_reports_queue.Size() < QUEUE_SIZE_LIMIT)
+	{
+		m_reports_queue.Push(report.Consume());
+		m_reporter_event.Set();
+	}
+}
+
+void AnalyticsReporter::ThreadProc()
+{
+	while (true)
+	{
+		m_reporter_event.Wait();
+		if (m_reporter_stop_request.IsSet())
+		{
+			return;
+		}
+
+		while (!m_reports_queue.Empty())
+		{
+			std::shared_ptr<AnalyticsReportingBackend> backend(m_backend);
+
+			if (backend)
+			{
+				std::string report;
+				m_reports_queue.Pop(report);
+				backend->Send(std::move(report));
+			}
+			else
+			{
+				break;
+			}
+
+			// Recheck after each report sent.
+			if (m_reporter_stop_request.IsSet())
+			{
+				return;
+			}
+		}
+	}
+}
+
+void StdoutAnalyticsBackend::Send(std::string report)
+{
+	printf("Analytics report sent:\n%s", HexDump(
+	    reinterpret_cast<const u8*>(report.data()), report.size()).c_str());
+}
+
+HttpAnalyticsBackend::HttpAnalyticsBackend(const std::string& endpoint)
+{
+	CURL* curl = curl_easy_init();
+	if (curl)
+	{
+		curl_easy_setopt(curl, CURLOPT_URL, endpoint.c_str());
+		curl_easy_setopt(curl, CURLOPT_POST, true);
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &DummyCurlWriteFunction);
+		curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, 3000);
+
+		// ALPN support is enabled by default but requires Windows >= 8.1.
+		curl_easy_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, false);
+
+		m_curl = curl;
+	}
+}
+
+HttpAnalyticsBackend::~HttpAnalyticsBackend()
+{
+	if (m_curl)
+	{
+		curl_easy_cleanup(m_curl);
+	}
+}
+
+void HttpAnalyticsBackend::Send(std::string report)
+{
+	if (!m_curl)
+		return;
+
+	curl_easy_setopt(m_curl, CURLOPT_POSTFIELDS, report.c_str());
+	curl_easy_setopt(m_curl, CURLOPT_POSTFIELDSIZE, report.size());
+	curl_easy_perform(m_curl);
+}
+
+}  // namespace Common

Source/Core/Common/Analytics.h

@@ -0,0 +1,194 @@
+// Copyright 2016 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "Common/CommonTypes.h"
+#include "Common/Event.h"
+#include "Common/FifoQueue.h"
+#include "Common/Flag.h"
+
+typedef void CURL;
+
+// Utilities for analytics reporting in Dolphin. This reporting is designed to
+// provide anonymous data about how well Dolphin performs in the wild. It also
+// allows developers to declare trace points in Dolphin's source code and get
+// information about what games trigger these conditions.
+//
+// This unfortunately implements Yet Another Serialization Framework within
+// Dolphin. We cannot really use ChunkFile because there is precedents for
+// backwards incompatible changes in the ChunkFile format. We could use
+// something like protobuf but setting up external dependencies is Hard™.
+//
+// Example usage:
+//
+// static auto s_reporter = std::make_unique<AnalyticsReporter>();
+// if (user_gave_consent)
+// {
+//     s_reporter->SetBackend(std::make_unique<MyReportingBackend>());
+// }
+// s_reporter->Send(s_reporter->Builder()
+//     .AddData("my_key", 42)
+//     .AddData("other_key", false));
+
+namespace Common
+{
+
+// Generic interface for an analytics reporting backends. The main
+// implementation used in Dolphin can be found in Core/Analytics.h.
+class AnalyticsReportingBackend
+{
+public:
+	virtual ~AnalyticsReportingBackend() {}
+
+	// Called from the AnalyticsReporter backend thread.
+	virtual void Send(std::string report) = 0;
+};
+
+// Builder object for an analytics report.
+class AnalyticsReportBuilder
+{
+public:
+	AnalyticsReportBuilder();
+	~AnalyticsReportBuilder() = default;
+
+	AnalyticsReportBuilder(const AnalyticsReportBuilder& other)
+	{
+		*this = other;
+	}
+
+	AnalyticsReportBuilder(AnalyticsReportBuilder&& other)
+	{
+		std::lock_guard<std::mutex> lk(other.m_lock);
+		m_report = std::move(other.m_report);
+	}
+
+	const AnalyticsReportBuilder& operator=(const AnalyticsReportBuilder& other)
+	{
+		if (this != &other)
+		{
+			std::lock_guard<std::mutex> lk(m_lock);
+			std::lock_guard<std::mutex> lk2(other.m_lock);
+			m_report = other.m_report;
+		}
+		return *this;
+	}
+
+	// Append another builder to this one.
+	AnalyticsReportBuilder& AddBuilder(const AnalyticsReportBuilder& other)
+	{
+		// Get before locking the object to avoid deadlocks with this += this.
+		std::string other_report = other.Get();
+		std::lock_guard<std::mutex> lk(m_lock);
+		m_report += other_report;
+		return *this;
+	}
+
+	template <typename T>
+	AnalyticsReportBuilder& AddData(const std::string& key, const T& value)
+	{
+		std::lock_guard<std::mutex> lk(m_lock);
+		AppendSerializedValue(&m_report, key);
+		AppendSerializedValue(&m_report, value);
+		return *this;
+	}
+
+	std::string Get() const
+	{
+		std::lock_guard<std::mutex> lk(m_lock);
+		return m_report;
+	}
+
+	// More efficient version of Get().
+	std::string Consume()
+	{
+		std::lock_guard<std::mutex> lk(m_lock);
+		return std::move(m_report);
+	}
+
+protected:
+	static void AppendSerializedValue(std::string* report, const std::string& v);
+	static void AppendSerializedValue(std::string* report, const char* v);
+	static void AppendSerializedValue(std::string* report, bool v);
+	static void AppendSerializedValue(std::string* report, u64 v);
+	static void AppendSerializedValue(std::string* report, s64 v);
+	static void AppendSerializedValue(std::string* report, u32 v);
+	static void AppendSerializedValue(std::string* report, s32 v);
+	static void AppendSerializedValue(std::string* report, float v);
+
+	// Should really be a std::shared_mutex, unfortunately that's C++17 only.
+	mutable std::mutex m_lock;
+	std::string m_report;
+};
+
+class AnalyticsReporter
+{
+public:
+	AnalyticsReporter();
+	~AnalyticsReporter();
+
+	// Sets a reporting backend and enables sending reports. Do not set a remote
+	// backend without user consent.
+	void SetBackend(std::unique_ptr<AnalyticsReportingBackend> backend)
+	{
+		m_backend = std::move(backend);
+		m_reporter_event.Set();  // In case reports are waiting queued.
+	}
+
+	// Gets the base report builder which is closed for each subsequent report
+	// being sent. DO NOT use this builder to send a report. Only use it to add
+	// new fields that should be globally available.
+	AnalyticsReportBuilder& BaseBuilder() { return m_base_builder; }
+
+	// Gets a cloned builder that can be used to send a report.
+	AnalyticsReportBuilder Builder() const { return m_base_builder; }
+
+	// Enqueues a report for sending. Consumes the report builder.
+	void Send(AnalyticsReportBuilder&& report);
+
+	// For convenience.
+	void Send(AnalyticsReportBuilder& report) { Send(std::move(report)); }
+
+protected:
+	void ThreadProc();
+
+	std::shared_ptr<AnalyticsReportingBackend> m_backend;
+	AnalyticsReportBuilder m_base_builder;
+
+	std::thread m_reporter_thread;
+	Common::Event m_reporter_event;
+	Common::Flag m_reporter_stop_request;
+	FifoQueue<std::string> m_reports_queue;
+};
+
+// Analytics backend to be used for debugging purpose, which dumps reports to
+// stdout.
+class StdoutAnalyticsBackend : public AnalyticsReportingBackend
+{
+public:
+	void Send(std::string report) override;
+};
+
+// Analytics backend that POSTs data to a remote HTTP(s) endpoint. WARNING:
+// remember to get explicit user consent before using.
+class HttpAnalyticsBackend : public AnalyticsReportingBackend
+{
+public:
+	HttpAnalyticsBackend(const std::string& endpoint);
+	~HttpAnalyticsBackend() override;
+
+	void Send(std::string report) override;
+
+protected:
+	CURL* m_curl = nullptr;
+};
+
+}  // namespace Common

Source/Core/Common/CMakeLists.txt

@@ -1,4 +1,5 @@
-set(SRCS BreakPoints.cpp
+set(SRCS Analytics.cpp
+         BreakPoints.cpp
          CDUtils.cpp
          ColorUtil.cpp
          ENetUtil.cpp
@@ -42,7 +43,7 @@ else()
 	         Logging/ConsoleListenerNix.cpp)
 endif()
 
-list(APPEND LIBS enet)
+list(APPEND LIBS enet ${CURL_LIBRARIES})
 if(_M_ARM_64)
 	set(SRCS ${SRCS}
 	         Arm64Emitter.cpp

Source/Core/Common/Common.vcxproj

@@ -35,6 +35,7 @@
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <ItemGroup>
+    <ClInclude Include="Analytics.h" />
     <ClInclude Include="Assert.h" />
     <ClInclude Include="Atomic.h" />
     <ClInclude Include="Atomic_GCC.h" />
@@ -141,6 +142,7 @@
     <ClInclude Include="Logging\LogManager.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="Analytics.cpp" />
     <ClCompile Include="BreakPoints.cpp" />
     <ClCompile Include="CDUtils.cpp" />
     <ClCompile Include="ColorUtil.cpp" />
@@ -194,6 +196,9 @@
     <ProjectReference Include="$(ExternalsDir)mbedtls\mbedTLS.vcxproj">
       <Project>{bdb6578b-0691-4e80-a46c-df21639fd3b8}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\..\..\Externals\curl\curl.vcxproj">
+      <Project>{bb00605c-125f-4a21-b33b-7bf418322dcb}</Project>
+    </ProjectReference>
     <ProjectReference Include="SCMRevGen.vcxproj">
       <Project>{41279555-f94f-4ebc-99de-af863c10c5c4}</Project>
     </ProjectReference>
@@ -204,4 +209,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

Source/Core/Common/Common.vcxproj.filters

@@ -224,6 +224,7 @@
     </ClInclude>
     <ClInclude Include="Assert.h" />
     <ClInclude Include="NonCopyable.h" />
+    <ClInclude Include="Analytics.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="BreakPoints.cpp" />
@@ -284,6 +285,7 @@
       <Filter>GL\GLInterface</Filter>
     </ClCompile>
     <ClCompile Include="ucrtFreadWorkaround.cpp" />
+    <ClCompile Include="Analytics.cpp" />
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />
@@ -291,4 +293,4 @@
   <ItemGroup>
     <Natvis Include="BitField.natvis" />
   </ItemGroup>
-</Project>
+</Project>