From 152b56c8cc20db4fb56a85d5749006d8d9d40aca Mon Sep 17 00:00:00 2001
From: Connor McLaughlin <stenzek@gmail.com>
Date: Tue, 10 Dec 2019 23:27:39 +1000
Subject: [PATCH] GPU: Fix mask bit setting on non-textured polygons

Fixes garbled colours in FMVs in some games.
---
 src/core/gpu_hw_shadergen.cpp | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/core/gpu_hw_shadergen.cpp b/src/core/gpu_hw_shadergen.cpp
index 09d306bc5..034457c38 100644
--- a/src/core/gpu_hw_shadergen.cpp
+++ b/src/core/gpu_hw_shadergen.cpp
@@ -512,6 +512,7 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
   bool semitransparent;
   int3 icolor;
   float ialpha;
+  float oalpha;
 
   #if TEXTURED
     #if TEXTURE_FILTERING
@@ -554,11 +555,17 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
     #else
       icolor = (vertcol * int3(texcol.rgb * float3(255.0, 255.0, 255.0))) >> 7;
     #endif
+
+    // Compute output alpha (mask bit)
+    oalpha = float(u_set_mask_while_drawing ? 1 : int(semitransparent));
   #else
     // All pixels are semitransparent for untextured polygons.
     semitransparent = true;
     icolor = vertcol;
     ialpha = 1.0;
+
+    // However, the mask bit is cleared if set mask bit is false.
+    oalpha = float(u_set_mask_while_drawing);
   #endif
 
   // Apply dithering
@@ -571,9 +578,6 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
     icolor = TruncateTo15Bit(icolor);
   #endif
 
-  // Compute output alpha (mask bit)
-  float output_alpha = float(u_set_mask_while_drawing ? 1 : int(semitransparent));
-
   // Normalize
   float3 color = float3(icolor) / float3(255.0, 255.0, 255.0);
 
@@ -586,7 +590,7 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
       #endif
 
       #if USE_DUAL_SOURCE
-        o_col0 = float4(color * (u_src_alpha_factor * ialpha), output_alpha);
+        o_col0 = float4(color * (u_src_alpha_factor * ialpha), oalpha);
         o_col1 = float4(0.0, 0.0, 0.0, u_dst_alpha_factor / ialpha);
       #else
         o_col0 = float4(color * (u_src_alpha_factor * ialpha), u_dst_alpha_factor / ialpha);
@@ -599,7 +603,7 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
       #endif
 
       #if USE_DUAL_SOURCE
-        o_col0 = float4(color * ialpha, output_alpha);
+        o_col0 = float4(color * ialpha, oalpha);
         o_col1 = float4(0.0, 0.0, 0.0, 0.0);
       #else
         o_col0 = float4(color * ialpha, 1.0 - ialpha);
@@ -607,7 +611,7 @@ float4 SampleFromVRAM(int4 texpage, int2 icoord)
     }
   #else
     // Non-transparency won't enable blending so we can write the mask here regardless.
-    o_col0 = float4(color * ialpha, output_alpha);
+    o_col0 = float4(color * ialpha, oalpha);
 
     #if USE_DUAL_SOURCE
       o_col1 = float4(0.0, 0.0, 0.0, 1.0 - ialpha);