From 4c9e16d5d4fb6f8ae1b826b9e5dd026f1bff39cc Mon Sep 17 00:00:00 2001
From: Jaklyy <102590697+Jaklyy@users.noreply.github.com>
Date: Fri, 16 Aug 2024 20:35:11 -0400
Subject: [PATCH] rework polygon orientation calcs

but this time actually correct
---
 src/GPU3D.cpp | 64 +++++++++++++++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/src/GPU3D.cpp b/src/GPU3D.cpp
index 865b00a1..ade5c417 100644
--- a/src/GPU3D.cpp
+++ b/src/GPU3D.cpp
@@ -965,50 +965,58 @@ void GPU3D::SubmitPolygon() noexcept
     VertexSlotCounter = 1;
     VertexSlotsFree = 0b11110;
 
-    // culling
+    // polygon orientation
     // TODO: work out how it works on the real thing
     // the normalization part is a wild guess
 
-    Vertex *v0, *v1, *v2, *v3;
+    Vertex *v0, *v1, *v2;
     s64 normalX, normalY, normalZ;
     s64 dot;
+    bool facingview;
 
     v0 = &TempVertexBuffer[0];
     v1 = &TempVertexBuffer[1];
     v2 = &TempVertexBuffer[2];
-    v3 = &TempVertexBuffer[3];
 
-    normalX = ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[3]-v1->Position[3]))
-        - ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[1]-v1->Position[1]));
-    normalY = ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[0]-v1->Position[0]))
-        - ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[3]-v1->Position[3]));
-    normalZ = ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[1]-v1->Position[1]))
-        - ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[0]-v1->Position[0]));
 
-    while ((((normalX>>31) ^ (normalX>>63)) != 0) ||
-           (((normalY>>31) ^ (normalY>>63)) != 0) ||
-           (((normalZ>>31) ^ (normalZ>>63)) != 0))
+    // setup vectors for v0->v2 and v1->v2, with w serving as the z axis
+    s32 vector0[3] = { v0->Position[0] - v2->Position[0], v0->Position[1] - v2->Position[1], v0->Position[3] - v2->Position[3] };
+    s32 vector1[3] = { v1->Position[0] - v2->Position[0], v1->Position[1] - v2->Position[1], v1->Position[3] - v2->Position[3] };
+
+    // if either vector is entirely 0, the polygon is accepted, and the front facing flag is set for the rasterizer
+    if (((vector0[0] | vector0[1] | vector0[2]) == 0) || ((vector1[0] | vector1[1] | vector1[2]) == 0))
     {
-        normalX >>= 4;
-        normalY >>= 4;
-        normalZ >>= 4;
+        facingview = true;
     }
-
-    dot = ((s64)v1->Position[0] * normalX) + ((s64)v1->Position[1] * normalY) + ((s64)v1->Position[3] * normalZ);
-
-    bool facingview = (dot <= 0);
-
-    if (dot < 0)
+    else
     {
-        if (!(CurPolygonAttr & (1<<7)))
+        // calculate cross product of the two vectors
+        normalX = ((s64)vector0[1] * vector1[2]) - ((s64)vector0[2] * vector1[1]);
+        normalY = ((s64)vector0[2] * vector1[0]) - ((s64)vector0[0] * vector1[2]);
+        normalZ = ((s64)vector0[0] * vector1[1]) - ((s64)vector0[1] * vector1[0]);
+
+        // normalization (currently a guess)
+        while ((((normalX>>31) ^ (normalX>>63)) != 0) ||
+               (((normalY>>31) ^ (normalY>>63)) != 0) ||
+               (((normalZ>>31) ^ (normalZ>>63)) != 0))
         {
-            LastStripPolygon = NULL;
-            return;
+            normalX >>= 4;
+            normalY >>= 4;
+            normalZ >>= 4;
         }
-    }
-    else if (dot > 0)
-    {
-        if (!(CurPolygonAttr & (1<<6)))
+        
+        // calculate dot product against the "camera vector" (we can use any of the vertices to do that)
+        dot = (v1->Position[0] * normalX) + (v1->Position[1] * normalY) + (v1->Position[3] * normalZ);
+
+        // front facing flag is set for the rasterizer to be used for:
+        // unwinding vertices
+        // determining whether slopes should be treated as "swapped"
+        // and whether to overwrite a pixel with equal depth when using the < depth test
+        facingview = (dot >= 0);
+
+        // cull polygon faces
+        if (!(((dot >= 0) && (CurPolygonAttr & (1<<7))) || // front facing
+              ((dot <= 0) && (CurPolygonAttr & (1<<6)))))  // back facing
         {
             LastStripPolygon = NULL;
             return;