vulkan_renderer: Alpha Test Culling Implementation

Used by various textures in many titles, e.g.  SSBU menu.

src/video_core/renderer_vulkan/fixed_pipeline_state.cpp

@@ -60,6 +60,11 @@ void FixedPipelineState::Fill(const Maxwell& regs, bool has_extended_dynamic_sta
     rasterize_enable.Assign(regs.rasterize_enable != 0 ? 1 : 0);
     topology.Assign(regs.draw.topology);
 
+    alpha_raw = 0;
+    alpha_test_enabled.Assign(regs.alpha_test_enabled);
+    alpha_test_func.Assign(PackComparisonOp(regs.alpha_test_func));
+    std::memcpy(&alpha_test_ref, &regs.alpha_test_ref, sizeof(u32)); // TODO: C++20 std::bit_cast
+
     std::memcpy(&point_size, &regs.point_size, sizeof(point_size)); // TODO: C++20 std::bit_cast
 
     for (std::size_t index = 0; index < Maxwell::NumVertexArrays; ++index) {

src/video_core/renderer_vulkan/fixed_pipeline_state.h

@@ -187,6 +187,14 @@ struct FixedPipelineState {
         BitField<23, 1, u32> rasterize_enable;
         BitField<24, 4, Maxwell::PrimitiveTopology> topology;
     };
+
+    u32 alpha_test_ref; /// < Alpha test reference
+    union {
+        u32 alpha_raw;
+        BitField<0, 3, u32> alpha_test_func;
+        BitField<3, 1, u32> alpha_test_enabled;
+    };
+
     u32 point_size;
     std::array<u32, Maxwell::NumVertexArrays> binding_divisors;
     std::array<VertexAttribute, Maxwell::NumVertexAttributes> attributes;

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -344,6 +344,14 @@ VKPipelineCache::DecompileShaders(const FixedPipelineState& fixed_state) {
     }
     specialization.ndc_minus_one_to_one = fixed_state.ndc_minus_one_to_one;
 
+    // Alpha test
+    if (fixed_state.alpha_test_enabled == 1) {
+        specialization.alpha_test_enabled = true;
+        specialization.alpha_test_func = static_cast<u8>(fixed_state.alpha_test_func);
+        // memcpy from u32 to float TODO: C++20 std::bit_cast
+        std::memcpy(&specialization.alpha_test_ref, &fixed_state.alpha_test_ref, sizeof(float));
+    }
+
     SPIRVProgram program;
     std::vector<VkDescriptorSetLayoutBinding> bindings;
 

src/video_core/renderer_vulkan/vk_shader_decompiler.cpp

@@ -2075,6 +2075,55 @@ private:
         return {};
     }
 
+    void AlphaTest(const Id& pointer) {
+        const Id true_label = OpLabel();
+        const Id skip_label = OpLabel();
+        Id condition;
+        switch (specialization.alpha_test_func) {
+        case VK_COMPARE_OP_NEVER:
+            condition = Constant(t_float, false); // Never true
+            break;
+        case VK_COMPARE_OP_LESS:
+            condition = OpFOrdLessThan(t_bool, Constant(t_float, specialization.alpha_test_ref),
+                                       OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_EQUAL:
+            condition = OpFOrdEqual(t_bool, Constant(t_float, specialization.alpha_test_ref),
+                                    OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_LESS_OR_EQUAL:
+            condition = OpFOrdLessThanEqual(
+                t_bool, Constant(t_float, specialization.alpha_test_ref), OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_GREATER:
+            // Note: requires "Equal" to properly work for ssbu. perhaps a precision issue
+            condition = OpFOrdGreaterThanEqual(
+                t_bool, Constant(t_float, specialization.alpha_test_ref), OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_NOT_EQUAL:
+            // Note: not accurate when tested against a unit test
+            // TODO: confirm if used by games
+            condition = OpFOrdNotEqual(t_bool, Constant(t_float, specialization.alpha_test_ref),
+                                       OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_GREATER_OR_EQUAL:
+            condition = OpFOrdGreaterThanEqual(
+                t_bool, Constant(t_float, specialization.alpha_test_ref), OpLoad(t_float, pointer));
+            break;
+        case VK_COMPARE_OP_ALWAYS:
+            condition = Constant(t_bool, true); // Always true
+            break;
+        default:
+            LOG_WARNING(Render_Vulkan, "Unimplemented alpha test function");
+            condition = Constant(t_bool, true); // Always true
+            break;
+        }
+        OpBranchConditional(condition, true_label, skip_label);
+        AddLabel(true_label);
+        OpKill();
+        AddLabel(skip_label);
+    }
+
     void PreExit() {
         if (stage == ShaderType::Vertex && specialization.ndc_minus_one_to_one) {
             const u32 position_index = out_indices.position.value();
@@ -2097,8 +2146,6 @@ private:
             UNIMPLEMENTED_IF_MSG(header.ps.omap.sample_mask != 0,
                                  "Sample mask write is unimplemented");
 
-            // TODO(Rodrigo): Alpha testing
-
             // Write the color outputs using the data in the shader registers, disabled
             // rendertargets/components are skipped in the register assignment.
             u32 current_reg = 0;
@@ -2110,6 +2157,9 @@ private:
                     }
                     const Id pointer = AccessElement(t_out_float, frag_colors[rt], component);
                     OpStore(pointer, SafeGetRegister(current_reg));
+                    if (specialization.alpha_test_enabled && component == 3) {
+                        AlphaTest(pointer);
+                    }
                     ++current_reg;
                 }
             }

src/video_core/renderer_vulkan/vk_shader_decompiler.h

@@ -95,6 +95,9 @@ struct Specialization final {
     std::bitset<Maxwell::NumVertexAttributes> enabled_attributes;
     std::array<Maxwell::VertexAttribute::Type, Maxwell::NumVertexAttributes> attribute_types{};
     bool ndc_minus_one_to_one{};
+    bool alpha_test_enabled{};
+    float alpha_test_ref{};
+    u8 alpha_test_func{};
 };
 // Old gcc versions don't consider this trivially copyable.
 // static_assert(std::is_trivially_copyable_v<Specialization>);