From ed79b1772c5cfff95346b69f8b2ba1ac355b1637 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Tue, 17 Oct 2023 18:53:50 -0400
Subject: [PATCH] invert coverage to fix xmajor aa

if the calculated y coordinate differs from the expected y coordinate, invert its coverage.
---
 src/GPU3D_Soft.cpp | 12 ++----------
 src/GPU3D_Soft.h   | 12 ++++++++++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/GPU3D_Soft.cpp b/src/GPU3D_Soft.cpp
index 03c6265e..c131dc0f 100644
--- a/src/GPU3D_Soft.cpp
+++ b/src/GPU3D_Soft.cpp
@@ -1070,11 +1070,7 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
     xlimit = xstart+l_edgelen;
     if (xlimit > xend+1) xlimit = xend+1;
     if (xlimit > 256) xlimit = 256;
-    if (l_edgecov & (1<<31))
-    {
-        xcov = (l_edgecov >> 12) & 0x3FF;
-        if (xcov == 0x3FF) xcov = 0;
-    }
+    if (l_edgecov & (1<<31)) xcov = (l_edgecov >> 12) & 0x3FF;
 
     if (!l_filledge) x = xlimit;
     else
@@ -1259,11 +1255,7 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
     edge = yedge | 0x2;
     xlimit = xend+1;
     if (xlimit > 256) xlimit = 256;
-    if (r_edgecov & (1<<31))
-    {
-        xcov = (r_edgecov >> 12) & 0x3FF;
-        if (xcov == 0x3FF) xcov = 0;
-    }
+    if (r_edgecov & (1<<31)) xcov = (r_edgecov >> 12) & 0x3FF;
 
     if (r_filledge)
     for (; x < xlimit; x++)
diff --git a/src/GPU3D_Soft.h b/src/GPU3D_Soft.h
index 2f5664e2..ec89cac5 100644
--- a/src/GPU3D_Soft.h
+++ b/src/GPU3D_Soft.h
@@ -236,6 +236,7 @@ private:
             dx = 0;
 
             this->x0 = x0;
+            this->y0 = y0;
             this->xmin = x0;
             this->xmax = x0;
 
@@ -253,6 +254,7 @@ private:
         constexpr s32 Setup(s32 x0, s32 x1, s32 y0, s32 y1, s32 w0, s32 w1, s32 y)
         {
             this->x0 = x0;
+            this->y0 = y0;
             this->y = y;
 
             if (x1 > x0)
@@ -367,7 +369,13 @@ private:
             if (side)     startx = startx - *length + 1;
 
             s32 startcov = (((startx << 10) + 0x1FF) * ylen) / xlen;
-            *coverage = (1<<31) | ((startcov & 0x3FF) << 12) | (xcov_incr & 0x3FF);
+
+            // fix the y value for negative slopes
+            s32 ycoord = Negative ? (ylen << 10) - startcov >> 10 : startcov >> 10;
+            // if yvalue is not equal to actual y value, invert coverage value
+            if (ycoord != y - y0) startcov = 0x3FF - (startcov & 0x3FF);
+
+            *coverage = (1<<31) | (startcov << 12) | (xcov_incr & 0x3FF);
 
             if constexpr (swapped) *length = 1;
         }
@@ -419,7 +427,7 @@ private:
         Interpolator<1> Interp;
 
     private:
-        s32 x0, xmin, xmax;
+        s32 x0, y0, xmin, xmax;
         s32 xlen, ylen;
         s32 dx;
         s32 y;