From 7ba7422ff843306c9cc02047e69ef8bfb7010db3 Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Wed, 30 Aug 2023 19:18:03 -0400
Subject: [PATCH] improve line polygon check

---
 src/GPU3D.cpp      | 69 +++++++++++++++++++++++++++++++---------------
 src/GPU3D_Soft.cpp |  4 +--
 2 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 049ae589..dcec7630 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -869,12 +869,47 @@ int ClipPolygon(GPU3D& gpu, Vertex* vertices, int nverts, int clipstart)
     return nverts;
 }
 
-bool ClipCoordsEqual(Vertex* a, Vertex* b)
+template<bool ogl>
+bool ClipCoordsEqual(Vertex a, Vertex b)
 {
-    return a->Position[0] == b->Position[0] &&
-           a->Position[1] == b->Position[1] &&
-           a->Position[2] == b->Position[2] &&
-           a->Position[3] == b->Position[3];
+    if constexpr (ogl)
+    return a.Position[0] == b.Position[0] &&
+           a.Position[1] == b.Position[1] &&
+           a.Position[2] == b.Position[2] &&
+           a.Position[3] == b.Position[3];
+    else
+    return a.FinalPosition[0] == b.FinalPosition[0] &&
+           a.FinalPosition[1] == b.FinalPosition[1];
+}
+
+template<bool ogl>
+bool LineCheck(Vertex* vertices, int nverts)
+{
+    // todo: check for more lines (perfectly vertical, horizontal or diagonal) for opengl renderer?
+    if (nverts == 3)
+    {
+        if (ClipCoordsEqual<ogl>(vertices[0], vertices[1]) ||
+            ClipCoordsEqual<ogl>(vertices[0], vertices[2]) ||
+            ClipCoordsEqual<ogl>(vertices[1], vertices[2]))
+        {
+            return true;
+        }
+    }
+    else if (nverts == 4)
+    {
+        int vertsequal = 0;
+
+        for (int a = 0; a < 3; a++)
+            for (int b = a+1; b < 4; b++)
+                if (ClipCoordsEqual<ogl>(vertices[a], vertices[b]))
+                {
+                    vertsequal++;
+
+                    if (vertsequal == 2)
+                        return true;
+                }
+    }
+    return false;
 }
 
 void GPU3D::SubmitPolygon() noexcept
@@ -992,21 +1027,7 @@ void GPU3D::SubmitPolygon() noexcept
         clippedvertices[i] = TempVertexBuffer[i];
 
     // detect lines, for the OpenGL renderer
-
-    int polytype = 0;
-    if (nverts == 3)
-    {
-        if (ClipCoordsEqual(&clippedvertices[0], &clippedvertices[1]) ||
-            ClipCoordsEqual(&clippedvertices[0], &clippedvertices[2]) ||
-            ClipCoordsEqual(&clippedvertices[1], &clippedvertices[2]))
-        {
-            polytype = 1;
-        }
-    }
-    else if (nverts == 4)
-    {
-        // TODO
-    }
+    int polytype = (GPU3D::CurrentRenderer->Accelerated) ? LineCheck<true>(clippedvertices, nverts) : 0;
 
     // clipping
 
@@ -1153,8 +1174,6 @@ void GPU3D::SubmitPolygon() noexcept
 
     if (!poly->Translucent) NumOpaquePolygons++;
 
-    poly->Type = polytype;
-
     if (LastStripPolygon && clipstart > 0)
     {
         if (nverts == lastpolyverts)
@@ -1275,6 +1294,12 @@ void GPU3D::SubmitPolygon() noexcept
         poly->FinalZ[i] = z;
         poly->FinalW[i] = w;
     }
+    
+    // check for line polygons for the software renderer
+    // checkme: can a line polygon have more than 4 or less than 3 vertices? 
+    polytype = (!GPU3D::CurrentRenderer->Accelerated) ? LineCheck<false>(*poly->Vertices, nverts) : polytype;
+    
+    poly->Type = polytype;
 
     if (PolygonMode >= 2)
         LastStripPolygon = poly;
diff --git a/src/GPU3D_Soft.cpp b/src/GPU3D_Soft.cpp
index 03c6265e..0003c58b 100644
--- a/src/GPU3D_Soft.cpp
+++ b/src/GPU3D_Soft.cpp
@@ -786,7 +786,7 @@ void SoftRenderer::RenderShadowMaskScanline(RendererPolygon* rp, s32 y)
         {
             l_filledge = ((rp->SlopeL.Negative || !rp->SlopeL.XMajor)
                 || (y == polygon->YBottom-1) && rp->SlopeL.XMajor && (vlnext->FinalPosition[0] != vrnext->FinalPosition[0]))
-                || (rp->SlopeL.Increment == rp->SlopeR.Increment) && (xstart+l_edgelen == xend+1);
+                || polygon->Type;
             r_filledge = (!rp->SlopeR.Negative && rp->SlopeR.XMajor) || (rp->SlopeR.Increment==0)
                 || (y == polygon->YBottom-1) && rp->SlopeR.XMajor && (vlnext->FinalPosition[0] != vrnext->FinalPosition[0]);
         }
@@ -1028,7 +1028,7 @@ void SoftRenderer::RenderPolygonScanline(RendererPolygon* rp, s32 y)
         {
             l_filledge = ((rp->SlopeL.Negative || !rp->SlopeL.XMajor)
                 || (y == polygon->YBottom-1) && rp->SlopeL.XMajor && (vlnext->FinalPosition[0] != vrnext->FinalPosition[0]))
-                || (rp->SlopeL.Increment == rp->SlopeR.Increment) && (xstart+l_edgelen == xend+1);
+                || polygon->Type;
             r_filledge = (!rp->SlopeR.Negative && rp->SlopeR.XMajor) || (rp->SlopeR.Increment==0)
                 || (y == polygon->YBottom-1) && rp->SlopeR.XMajor && (vlnext->FinalPosition[0] != vrnext->FinalPosition[0]);
         }