From 0c8095c1bb0e43ce4db884500b372aeb853d0ca4 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Wed, 18 Oct 2023 00:36:53 -0400
Subject: [PATCH] rewrite of aa code

should behave identically
---
 src/GPU3D_Soft.cpp | 24 ++++++++----------------
 src/GPU3D_Soft.h   | 31 +++++++++++--------------------
 2 files changed, 19 insertions(+), 36 deletions(-)

diff --git a/src/GPU3D_Soft.cpp b/src/GPU3D_Soft.cpp
index c131dc0f..1871caeb 100644
--- a/src/GPU3D_Soft.cpp
+++ b/src/GPU3D_Soft.cpp
@@ -1063,14 +1063,11 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
     if (x < 0) x = 0;
     s32 xlimit;
 
-    s32 xcov = 0;
-
     // part 1: left edge
     edge = yedge | 0x1;
     xlimit = xstart+l_edgelen;
     if (xlimit > xend+1) xlimit = xend+1;
     if (xlimit > 256) xlimit = 256;
-    if (l_edgecov & (1<<31)) xcov = (l_edgecov >> 12) & 0x3FF;
 
     if (!l_filledge) x = xlimit;
     else
@@ -1129,14 +1126,12 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
                 // anti-aliasing: all edges are rendered
 
                 // calculate coverage
-                s32 cov = l_edgecov;
-                if (cov & (1<<31))
+                attr |= (l_edgecov >> 5) << 8;
+                if (x < xlimit-1)
                 {
-                    cov = xcov >> 5;
-                    if (cov > 31) cov = 31;
-                    xcov += (l_edgecov & 0x3FF);
+                    l_edgecov += rp->SlopeL.XCov_Incr;
+                    if (l_edgecov > 0x3FF) l_edgecov = 0x3FF;
                 }
-                attr |= (cov << 8);
 
                 // push old pixel down if needed
                 if (pixeladdr < BufferSize)
@@ -1255,7 +1250,6 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
     edge = yedge | 0x2;
     xlimit = xend+1;
     if (xlimit > 256) xlimit = 256;
-    if (r_edgecov & (1<<31)) xcov = (r_edgecov >> 12) & 0x3FF;
 
     if (r_filledge)
     for (; x < xlimit; x++)
@@ -1313,14 +1307,12 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
                 // anti-aliasing: all edges are rendered
 
                 // calculate coverage
-                s32 cov = r_edgecov;
-                if (cov & (1<<31))
+                attr |= (r_edgecov >> 5) << 8;
+                if (x < xlimit-1)
                 {
-                    cov = 0x1F - (xcov >> 5);
-                    if (cov < 0) cov = 0;
-                    xcov += (r_edgecov & 0x3FF);
+                    r_edgecov -= rp->SlopeR.XCov_Incr;
+                    if (r_edgecov < 0) r_edgecov = 0;
                 }
-                attr |= (cov << 8);
 
                 // push old pixel down if needed
                 if (pixeladdr < BufferSize)
diff --git a/src/GPU3D_Soft.h b/src/GPU3D_Soft.h
index ec89cac5..38de794c 100644
--- a/src/GPU3D_Soft.h
+++ b/src/GPU3D_Soft.h
@@ -246,7 +246,7 @@ private:
             Interp.Setup(0, 0, 0, 0);
             Interp.SetX(0);
 
-            xcov_incr = 0;
+            XCov_Incr = 0;
 
             return x0;
         }
@@ -322,7 +322,7 @@ private:
             Interp.SetX(y);
 
             // used for calculating AA coverage
-            if (XMajor) xcov_incr = (ylen << 10) / xlen;
+            if (XMajor) XCov_Incr = ((ylen << 10) / xlen) & 0x3FF;
 
             return x;
         }
@@ -373,9 +373,11 @@ private:
             // fix the y value for negative slopes
             s32 ycoord = Negative ? (ylen << 10) - startcov >> 10 : startcov >> 10;
             // if yvalue is not equal to actual y value, invert coverage value
-            if (ycoord != y - y0) startcov = 0x3FF - (startcov & 0x3FF);
+            startcov &= 0x3FF;
+            if (ycoord != y - y0) startcov = 0x3FF - startcov;
+            if (side ^ swapped) startcov = 0x3FF - startcov;
 
-            *coverage = (1<<31) | (startcov << 12) | (xcov_incr & 0x3FF);
+            *coverage = startcov;
 
             if constexpr (swapped) *length = 1;
         }
@@ -389,26 +391,17 @@ private:
             {
                 // for some reason vertical edges' aa values
                 // are inverted too when the edges are swapped
-                if constexpr (swapped)
-                    *coverage = 0;
-                else
-                    *coverage = 31;
+                *coverage = swapped ? 0 : 31 << 5;
             }
             else
             {
                 s32 cov = ((dx >> 9) + (Increment >> 10)) >> 4;
                 if ((cov >> 5) != (dx >> 18)) cov = 31;
                 cov &= 0x1F;
-                if constexpr (swapped)
-                {
-                    if (side ^ Negative) cov = 0x1F - cov;
-                }
-                else
-                {
-                    if (!(side ^ Negative)) cov = 0x1F - cov;
-                }
+                if (side ^ !Negative ^ swapped) cov = 0x1F - cov;
 
-                *coverage = cov;
+                // shift left 5 just to make it align with xmajor coverage values
+                *coverage = cov << 5;
             }
         }
 
@@ -425,15 +418,13 @@ private:
         bool Negative;
         bool XMajor;
         Interpolator<1> Interp;
+        s32 XCov_Incr;
 
     private:
         s32 x0, y0, xmin, xmax;
         s32 xlen, ylen;
         s32 dx;
         s32 y;
-
-        s32 xcov_incr;
-        s32 ycoverage, ycov_incr;
     };
 
     template <typename T>