Merge pull request #6348 from stenzek/fog

BPMemory: Handle fog configuration where both A and C are infinity/NaN

Source/Core/VideoBackends/Software/Tev.cpp

@@ -751,14 +751,14 @@ void Tev::Draw()
       // ze = A/(B - (Zs >> B_SHF))
       const s32 denom = bpmem.fog.b_magnitude - (Position[2] >> bpmem.fog.b_shift);
       // in addition downscale magnitude and zs to 0.24 bits
-      ze = (bpmem.fog.a.GetA() * 16777215.0f) / (float)denom;
+      ze = (bpmem.fog.GetA() * 16777215.0f) / static_cast<float>(denom);
     }
     else
     {
       // orthographic
       // ze = a*Zs
       // in addition downscale zs to 0.24 bits
-      ze = bpmem.fog.a.GetA() * ((float)Position[2] / 16777215.0f);
+      ze = bpmem.fog.GetA() * (static_cast<float>(Position[2]) / 16777215.0f);
     }
 
     if (bpmem.fogRange.Base.Enabled)
@@ -796,7 +796,7 @@ void Tev::Draw()
                        // GXInitFogAdjTable): 1/cos = c/b = sqrt(a^2+b^2)/b
     }
 
-    ze -= bpmem.fog.c_proj_fsel.GetC();
+    ze -= bpmem.fog.GetC();
 
     // clamp 0 to 1
     float fog = (ze < 0.0f) ? 0.0f : ((ze > 1.0f) ? 1.0f : ze);

Source/Core/VideoCommon/BPMemory.cpp

@@ -23,21 +23,55 @@ bool BlendMode::UseLogicOp() const
   return true;
 }
 
-float FogParam0::GetA() const
+bool FogParams::IsNaNCase() const
 {
+  // Check for the case where both a and c are infinity or NaN.
+  // On hardware, this results in the following colors:
+  //
+  // -------------------------------------------------------
+  // |   A   |   C   |  Result  |   A   |   C   |  Result  |
+  // -------------------------------------------------------
+  // |  inf  |  inf  |  Fogged  |  inf  |  nan  |  Fogged  |
+  // |  inf  | -inf  | Unfogged |  inf  | -nan  | Unfogged |
+  // | -inf  |  inf  | Unfogged | -inf  |  nan  | Unfogged |
+  // | -inf  | -inf  | Unfogged | -inf  | -nan  | Unfogged |
+  // -------------------------------------------------------
+  // |  nan  |  inf  |  Fogged  |  nan  |  nan  |  Fogged  |
+  // |  nan  | -inf  | Unfogged |  nan  | -nan  | Unfogged |
+  // | -nan  |  inf  | Unfogged | -nan  |  nan  | Unfogged |
+  // | -nan  | -inf  | Unfogged | -nan  | -nan  | Unfogged |
+  // -------------------------------------------------------
+  //
+  // We replicate this by returning A=0, and C=inf for the inf/inf case, otherwise -inf.
+  // This ensures we do not pass a NaN to the GPU, and -inf/inf clamp to 0/1 respectively.
+  return a.exp == 255 && c_proj_fsel.c_exp == 255;
+}
+
+float FogParams::GetA() const
+{
+  if (IsNaNCase())
+    return 0.0f;
+
   // scale mantissa from 11 to 23 bits
-  const u32 integral = (static_cast<u32>(sign) << 31) | (static_cast<u32>(exponent) << 23) |
+  const u32 integral = (static_cast<u32>(a.sign) << 31) | (static_cast<u32>(a.exp) << 23) |
-                       (static_cast<u32>(mantissa) << 12);
+                       (static_cast<u32>(a.mant) << 12);
 
   float real;
   std::memcpy(&real, &integral, sizeof(u32));
   return real;
 }
 
-float FogParam3::GetC() const
+float FogParams::GetC() const
 {
+  if (IsNaNCase())
+  {
+    constexpr float inf = std::numeric_limits<float>::infinity();
+    return !a.sign && !c_proj_fsel.c_sign ? -inf : inf;
+  }
+
   // scale mantissa from 11 to 23 bits
-  const u32 integral = (c_sign.Value() << 31) | (c_exp.Value() << 23) | (c_mant.Value() << 12);
+  const u32 integral = (c_proj_fsel.c_sign.Value() << 31) | (c_proj_fsel.c_exp.Value() << 23) |
+                       (c_proj_fsel.c_mant.Value() << 12);
 
   float real;
   std::memcpy(&real, &integral, sizeof(u32));

Source/Core/VideoCommon/BPMemory.h

@@ -654,14 +654,9 @@ union BlendMode
 
 union FogParam0
 {
-  struct
+  BitField<0, 11, u32> mant;
-  {
+  BitField<11, 8, u32> exp;
-    u32 mantissa : 11;
+  BitField<19, 1, u32> sign;
-    u32 exponent : 8;
-    u32 sign : 1;
-  };
-
-  float GetA() const;
 
   u32 hex;
 };
@@ -675,9 +670,6 @@ union FogParam3
   BitField<21, 3, u32> fsel;  // 0 - off, 2 - linear, 4 - exp, 5 - exp2, 6 -
                               // backward exp, 7 - backward exp2
 
-  // amount to subtract from eyespacez after range adjustment
-  float GetC() const;
-
   u32 hex;
 };
 
@@ -721,6 +713,13 @@ struct FogParams
   };
 
   FogColor color;  // 0:b 8:g 16:r - nice!
+
+  // Special case where a and c are infinite and the sign matches, resulting in a result of NaN.
+  bool IsNaNCase() const;
+  float GetA() const;
+
+  // amount to subtract from eyespacez after range adjustment
+  float GetC() const;
 };
 
 union ZMode

Source/Core/VideoCommon/PixelShaderManager.cpp

@@ -409,9 +409,9 @@ void PixelShaderManager::SetFogParamChanged()
 {
   if (!g_ActiveConfig.bDisableFog)
   {
-    constants.fogf[1][0] = bpmem.fog.a.GetA();
+    constants.fogf[1][0] = bpmem.fog.GetA();
     constants.fogi[1] = bpmem.fog.b_magnitude;
-    constants.fogf[1][2] = bpmem.fog.c_proj_fsel.GetC();
+    constants.fogf[1][2] = bpmem.fog.GetC();
     constants.fogi[3] = bpmem.fog.b_shift;
     constants.fogParam3 = bpmem.fog.c_proj_fsel.hex;
   }