Fix aa being upside down on swapped y-major slopes (#1803)

* fix aa being upside down on swapped y-major slopes

* further improvements to swapped aa

in addition to fixing swapped y-major slope aa, now fixes:
swapped x-major slope aa
swapped vertical slope aa

* use templates instead + style/comment tweaks

should force the compiler to precompile if statements like i want it to do, instead of just hoping it does so on its own

src/GPU3D_Soft.cpp

@@ -753,8 +753,8 @@ void SoftRenderer::RenderShadowMaskScanline(RendererPolygon* rp, s32 y)
         interp_start = &rp->SlopeR.Interp;
         interp_end = &rp->SlopeL.Interp;
 
-        rp->SlopeR.EdgeParams_YMajor(&l_edgelen, &l_edgecov);
+        rp->SlopeR.EdgeParams<true>(&l_edgelen, &l_edgecov);
-        rp->SlopeL.EdgeParams_YMajor(&r_edgelen, &r_edgecov);
+        rp->SlopeL.EdgeParams<true>(&r_edgelen, &r_edgecov);
 
         std::swap(xstart, xend);
         std::swap(wl, wr);
@@ -771,8 +771,8 @@ void SoftRenderer::RenderShadowMaskScanline(RendererPolygon* rp, s32 y)
         interp_start = &rp->SlopeL.Interp;
         interp_end = &rp->SlopeR.Interp;
 
-        rp->SlopeL.EdgeParams(&l_edgelen, &l_edgecov);
+        rp->SlopeL.EdgeParams<false>(&l_edgelen, &l_edgecov);
-        rp->SlopeR.EdgeParams(&r_edgelen, &r_edgecov);
+        rp->SlopeR.EdgeParams<false>(&r_edgelen, &r_edgecov);
     }
 
     // color/texcoord attributes aren't needed for shadow masks
@@ -958,10 +958,8 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
 
     // if the left and right edges are swapped, render backwards.
     // on hardware, swapped edges seem to break edge length calculation,
-    // causing X-major edges to be rendered wrong when
+    // causing X-major edges to be rendered wrong when filled,
-    // wireframe/edgemarking/antialiasing are used
+    // and resulting in buggy looking anti-aliasing on X-major edges
-    // it also causes bad antialiasing, but not sure what's going on (TODO)
-    // most probable explanation is that such slopes are considered to be Y-major
 
     if (xstart > xend)
     {
@@ -973,8 +971,8 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
         interp_start = &rp->SlopeR.Interp;
         interp_end = &rp->SlopeL.Interp;
 
-        rp->SlopeR.EdgeParams_YMajor(&l_edgelen, &l_edgecov);
+        rp->SlopeR.EdgeParams<true>(&l_edgelen, &l_edgecov);
-        rp->SlopeL.EdgeParams_YMajor(&r_edgelen, &r_edgecov);
+        rp->SlopeL.EdgeParams<true>(&r_edgelen, &r_edgecov);
 
         std::swap(xstart, xend);
         std::swap(wl, wr);
@@ -991,8 +989,8 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
         interp_start = &rp->SlopeL.Interp;
         interp_end = &rp->SlopeR.Interp;
 
-        rp->SlopeL.EdgeParams(&l_edgelen, &l_edgecov);
+        rp->SlopeL.EdgeParams<false>(&l_edgelen, &l_edgecov);
-        rp->SlopeR.EdgeParams(&r_edgelen, &r_edgecov);
+        rp->SlopeR.EdgeParams<false>(&r_edgelen, &r_edgecov);
     }
 
     // interpolate attributes along Y

src/GPU3D_Soft.h

@@ -354,13 +354,19 @@ private:
             else if (ret > xmax) ret = xmax;
             return ret;
         }
-
+        
+        template<bool swapped>
         void EdgeParams_XMajor(s32* length, s32* coverage)
         {
-            if (side ^ Negative)
+            // only do length calc for right side when swapped as it's
-                *length = (dx >> 18) - ((dx-Increment) >> 18);
+            // only needed for aa calcs, as actual line spans are broken
-            else
+            if constexpr (!swapped || side)
-                *length = ((dx+Increment) >> 18) - (dx >> 18);
+            {
+                if (side ^ Negative)
+                    *length = (dx >> 18) - ((dx-Increment) >> 18);
+                else
+                    *length = ((dx+Increment) >> 18) - (dx >> 18);
+            }
 
             // for X-major edges, we return the coverage
             // for the first pixel, and the increment for
@@ -371,33 +377,49 @@ private:
 
             s32 startcov = (((startx << 10) + 0x1FF) * ylen) / xlen;
             *coverage = (1<<31) | ((startcov & 0x3FF) << 12) | (xcov_incr & 0x3FF);
+
+            if constexpr (swapped) *length = 1;
         }
 
+        template<bool swapped>
         void EdgeParams_YMajor(s32* length, s32* coverage)
         {
             *length = 1;
 
             if (Increment == 0)
             {
-                *coverage = 31;
+                // for some reason vertical edges' aa values
+                // are inverted too when the edges are swapped
+                if constexpr (swapped)
+                    *coverage = 0;
+                else
+                    *coverage = 31;
             }
             else
             {
                 s32 cov = ((dx >> 9) + (Increment >> 10)) >> 4;
                 if ((cov >> 5) != (dx >> 18)) cov = 31;
                 cov &= 0x1F;
-                if (!(side ^ Negative)) cov = 0x1F - cov;
+                if constexpr (swapped)
+                {
+                    if (side ^ Negative) cov = 0x1F - cov;
+                }
+                else
+                {
+                    if (!(side ^ Negative)) cov = 0x1F - cov;
+                }
 
                 *coverage = cov;
             }
         }
-
+        
+        template<bool swapped>
         void EdgeParams(s32* length, s32* coverage)
         {
             if (XMajor)
-                return EdgeParams_XMajor(length, coverage);
+                return EdgeParams_XMajor<swapped>(length, coverage);
             else
-                return EdgeParams_YMajor(length, coverage);
+                return EdgeParams_YMajor<swapped>(length, coverage);
         }
 
         s32 Increment;