ClearScreen fixes:

- Clear alpha channel to 0xFF instead of 0x00 whenever we (re-)create the EFB texture
- Disable color/alpha channels whenever the actual EFB format doesn't use them

Fixes issue 2826.
Fixes issue 1879.

Please check if issue 3014 is fixed now, as well as any other games which might have broken due to ClearScreen changes.
For what it's worth, Super Mario Sunshine still shows a small glitch even with these changes.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@6635 8ced0084-cf51-0410-be5f-012b33b47a6e

Source/Core/VideoCommon/Src/BPFunctions.cpp

@@ -25,7 +25,6 @@
 
 bool textureChanged[8];
 const bool renderFog = false;
-u32 prev_pix_format = -1;
 
 namespace BPFunctions
 {
@@ -90,7 +89,7 @@ void CopyEFB(const BPCmd &bp, const EFBRectangle &rc, const u32 &address, const
 /* Explanation of the magic behind ClearScreen:
 	There's numerous possible formats for the pixel data in the EFB.
 	However, in the HW accelerated plugins we're always using RGBA8
-	for the EFB format, which implicates some problems:
+	for the EFB format, which causes some problems:
 	- We're using an alpha channel although the game doesn't (1)
 	- If the actual EFB format is PIXELFMT_RGBA6_Z24, we are using more bits per channel than the native HW (2)
 	- When doing a z copy (EFB copy target format GX_TF_Z24X8 (and possibly others?)), the native HW assumes that the EFB format is RGB8 when clearing.
@@ -101,10 +100,9 @@ void CopyEFB(const BPCmd &bp, const EFBRectangle &rc, const u32 &address, const
 
 	To properly emulate the above points, we're doing the following:
 	(1)
-		- disable alpha channel writing in any kind of rendering if the actual EFB format doesn't use an alpha channel
-		- when clearing:
-			- if the actual EFB format uses an alpha channel, enable alpha channel writing if alphaupdate is set
-			- if the actual EFB format doesn't use an alpha channel, set the alpha channel to 0xFF
+		- disable alpha channel writing of any kind of rendering if the actual EFB format doesn't use an alpha channel
+		- NOTE: Always make sure that the EFB has been cleared to an alpha value of 0xFF in this case!
+		- Same for color channels, these need to be cleared to 0x00 though.
 	(2)
 		- just scale down the RGBA8 color to RGBA6 and upscale it to RGBA8 again
 	(3)
@@ -117,17 +115,39 @@ void CopyEFB(const BPCmd &bp, const EFBRectangle &rc, const u32 &address, const
 */
 void ClearScreen(const BPCmd &bp, const EFBRectangle &rc)
 {
+	UPE_Copy PE_copy = bpmem.triggerEFBCopy;
 	bool colorEnable = bpmem.blendmode.colorupdate;
-	bool alphaEnable = bpmem.blendmode.alphaupdate || (bpmem.zcontrol.pixel_format != PIXELFMT_RGBA6_Z24); // (1)
+	bool alphaEnable = bpmem.blendmode.alphaupdate;
 	bool zEnable = bpmem.zmode.updateenable;
 
+	// (1): Disable unused color channels
+	switch (bpmem.zcontrol.pixel_format)
+	{
+		case PIXELFMT_RGBA6_Z24:
+			if (colorEnable && PE_copy.tp_realFormat() == GX_TF_Z24X8) // (3): alpha update forced
+				alphaEnable = true;
+			break;
+
+		case PIXELFMT_RGB8_Z24:
+		case PIXELFMT_RGB565_Z16:
+			alphaEnable = false;
+			break;
+
+		case PIXELFMT_Z24:
+			alphaEnable = colorEnable = false;
+			break;
+
+		default:
+			// TODO?
+			break;
+	}
+
 	if (colorEnable || alphaEnable || zEnable)
 	{
 		u32 color = (bpmem.clearcolorAR << 16) | bpmem.clearcolorGB;
 		u32 z = bpmem.clearZValue;
 		if (bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24)
 		{
-			UPE_Copy PE_copy = bpmem.triggerEFBCopy;
 			// TODO: Not sure whether there's more formats to check for here - maybe GX_TF_Z8 and GX_TF_Z16?
 			if (PE_copy.tp_realFormat() == GX_TF_Z24X8) // (3): Reinterpret RGB8 color as RGBA6
 			{
@@ -150,11 +170,11 @@ void ClearScreen(const BPCmd &bp, const EFBRectangle &rc)
 				color |= (color >> 6) & 0x3030303;
 			}
 		}
-		else // (1): Clear alpha channel to 0xFF if no alpha channel is supposed to be there
+		if (bpmem.zcontrol.pixel_format == PIXELFMT_RGB565_Z16)
 		{
-			color |= (prev_pix_format == PIXELFMT_RGBA6_Z24) ? 0x0 : 0xFF000000;
+			z >>=8;
+			z = z | (z>>16);
 		}
-		prev_pix_format = bpmem.zcontrol.pixel_format;
 		g_renderer->ClearScreen(rc, colorEnable, alphaEnable, zEnable, color, z);
 	}
 }

Source/Plugins/Plugin_VideoDX11/Src/Render.cpp

@@ -341,7 +341,7 @@ Renderer::Renderer()
 	for (unsigned int stage = 0; stage < 8; stage++)
 		D3D::gfxstate->samplerdesc[stage].MaxAnisotropy = g_ActiveConfig.iMaxAnisotropy;
 
-	float ClearColor[4] = { 0.f, 0.f, 0.f, 0.f };
+	float ClearColor[4] = { 0.f, 0.f, 0.f, 1.f };
 	D3D::context->ClearRenderTargetView(FramebufferManager::GetEFBColorTexture()->GetRTV(), ClearColor);
 	D3D::context->ClearDepthStencilView(FramebufferManager::GetEFBDepthTexture()->GetDSV(), D3D11_CLEAR_DEPTH, 1.f, 0);
 
@@ -466,7 +466,7 @@ void Renderer::SetColorMask()
 	UINT8 color_mask = 0;
 	if (bpmem.blendmode.alphaupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24))
 		color_mask = D3D11_COLOR_WRITE_ENABLE_ALPHA;
-	if (bpmem.blendmode.colorupdate)
+	if (bpmem.blendmode.colorupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGB8_Z24 || bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24 || bpmem.zcontrol.pixel_format == PIXELFMT_RGB565_Z16))
 		color_mask |= D3D11_COLOR_WRITE_ENABLE_RED | D3D11_COLOR_WRITE_ENABLE_GREEN | D3D11_COLOR_WRITE_ENABLE_BLUE;
 	D3D::gfxstate->SetRenderTargetWriteMask(color_mask);
 }
@@ -677,6 +677,9 @@ void Renderer::UpdateViewport()
 			g_framebuffer_manager = new FramebufferManager;
 
 			D3D::context->OMSetRenderTargets(1, &FramebufferManager::GetEFBColorTexture()->GetRTV(), FramebufferManager::GetEFBDepthTexture()->GetDSV());
+			float clear_col[4] = { 0.f, 0.f, 0.f, 1.f };
+			D3D::context->ClearRenderTargetView(FramebufferManager::GetEFBColorTexture()->GetRTV(), clear_col);
+			D3D::context->ClearDepthStencilView(FramebufferManager::GetEFBDepthTexture()->GetDSV(), D3D11_CLEAR_DEPTH, 1.f, 0);
 		}
 	}
 
@@ -978,6 +981,9 @@ void Renderer::Swap(u32 xfbAddr, FieldType field, u32 fbWidth, u32 fbHeight,cons
 		
 		delete g_framebuffer_manager;
 		g_framebuffer_manager = new FramebufferManager;
+		float clear_col[4] = { 0.f, 0.f, 0.f, 1.f };
+		D3D::context->ClearRenderTargetView(FramebufferManager::GetEFBColorTexture()->GetRTV(), clear_col);
+		D3D::context->ClearDepthStencilView(FramebufferManager::GetEFBDepthTexture()->GetDSV(), D3D11_CLEAR_DEPTH, 1.f, 0);
 	}
 
 	// begin next frame

Source/Plugins/Plugin_VideoDX9/Src/Render.cpp

@@ -322,7 +322,7 @@ Renderer::Renderer()
 	vp.Width  = s_target_width;
 	vp.Height = s_target_height;
 	D3D::dev->SetViewport(&vp);
-	D3D::dev->Clear(0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, 0x0, 1.0f, 0);
+	D3D::dev->Clear(0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, D3DCOLOR_ARGB(255,0,0,0), 1.0f, 0);
 	D3D::BeginFrame();
 	D3D::SetRenderState(D3DRS_SCISSORTESTENABLE, true);
 	D3D::dev->CreateOffscreenPlainSurface(s_backbuffer_width,s_backbuffer_height, D3DFMT_X8R8G8B8, D3DPOOL_SYSTEMMEM, &ScreenShootMEMSurface, NULL );
@@ -477,7 +477,7 @@ void Renderer::SetColorMask()
 	DWORD color_mask = 0;
 	if (bpmem.blendmode.alphaupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24))
 		color_mask = D3DCOLORWRITEENABLE_ALPHA;
-	if (bpmem.blendmode.colorupdate)
+	if (bpmem.blendmode.colorupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGB565_Z16 || bpmem.zcontrol.pixel_format == PIXELFMT_RGB8_Z24|| bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24))
 		color_mask |= D3DCOLORWRITEENABLE_RED | D3DCOLORWRITEENABLE_GREEN | D3DCOLORWRITEENABLE_BLUE;
 	D3D::SetRenderState(D3DRS_COLORWRITEENABLE, color_mask);
 }
@@ -771,6 +771,7 @@ void Renderer::UpdateViewport()
 
 			D3D::dev->SetRenderTarget(0, FramebufferManager::GetEFBColorRTSurface());
 			D3D::dev->SetDepthStencilSurface(FramebufferManager::GetEFBDepthRTSurface());
+			D3D::dev->Clear(0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, D3DCOLOR_ARGB(255,0,0,0), 1.0f, 0);
 		}
 	}
 
@@ -1159,6 +1160,7 @@ void Renderer::Swap(u32 xfbAddr, FieldType field, u32 fbWidth, u32 fbHeight,cons
 		}
 		D3D::dev->SetRenderTarget(0, FramebufferManager::GetEFBColorRTSurface());
 		D3D::dev->SetDepthStencilSurface(FramebufferManager::GetEFBDepthRTSurface());
+		D3D::dev->Clear(0, NULL, D3DCLEAR_TARGET | D3DCLEAR_ZBUFFER, D3DCOLOR_ARGB(255,0,0,0), 1.0f, 0);
 	}
 
 	// Place messages on the picture, then copy it to the screen

Source/Plugins/Plugin_VideoOGL/Src/Render.cpp

@@ -662,8 +662,11 @@ bool Renderer::SetScissorRect()
 void Renderer::SetColorMask()
 {
 	// Only enable alpha channel if it's supported by the current EFB format
-	GLenum ColorMask = (bpmem.blendmode.colorupdate) ? GL_TRUE : GL_FALSE;
-	GLenum AlphaMask = (bpmem.blendmode.alphaupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24)) ? GL_TRUE : GL_FALSE;
+	GLenum ColorMask = GL_FALSE, AlphaMask = GL_FALSE;
+	if (bpmem.blendmode.colorupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGB8_Z24 || bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24 || bpmem.zcontrol.pixel_format == PIXELFMT_RGB565_Z16))
+		AlphaMask = GL_TRUE;
+	if (bpmem.blendmode.alphaupdate && (bpmem.zcontrol.pixel_format == PIXELFMT_RGBA6_Z24))
+		ColorMask = GL_TRUE;
 	glColorMask(ColorMask,  ColorMask,  ColorMask,  AlphaMask);
 }