3 files changed, 140 insertions, 97 deletions

diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
index 97aab951a..8734045e5 100644
--- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
+++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
@@ -12,23 +12,32 @@
 
 namespace Vulkan {
 
-namespace {
-
-constexpr FixedPipelineState::DepthStencil GetDepthStencilState(const Maxwell& regs) {
-    const FixedPipelineState::StencilFace front_stencil(
-        regs.stencil_front_op_fail, regs.stencil_front_op_zfail, regs.stencil_front_op_zpass,
-        regs.stencil_front_func_func);
-    const FixedPipelineState::StencilFace back_stencil =
-        regs.stencil_two_side_enable
-            ? FixedPipelineState::StencilFace(regs.stencil_back_op_fail, regs.stencil_back_op_zfail,
-                                              regs.stencil_back_op_zpass,
-                                              regs.stencil_back_func_func)
-            : front_stencil;
-    return FixedPipelineState::DepthStencil(
-        regs.depth_test_enable == 1, regs.depth_write_enabled == 1, regs.depth_bounds_enable == 1,
-        regs.stencil_enable == 1, regs.depth_test_func, front_stencil, back_stencil);
+void FixedPipelineState::DepthStencil::Fill(const Maxwell& regs) noexcept {
+    raw = 0;
+    front.action_stencil_fail.Assign(PackStencilOp(regs.stencil_front_op_fail));
+    front.action_depth_fail.Assign(PackStencilOp(regs.stencil_front_op_zfail));
+    front.action_depth_pass.Assign(PackStencilOp(regs.stencil_front_op_zpass));
+    front.test_func.Assign(PackComparisonOp(regs.stencil_front_func_func));
+    if (regs.stencil_two_side_enable) {
+        back.action_stencil_fail.Assign(PackStencilOp(regs.stencil_back_op_fail));
+        back.action_depth_fail.Assign(PackStencilOp(regs.stencil_back_op_zfail));
+        back.action_depth_pass.Assign(PackStencilOp(regs.stencil_back_op_zpass));
+        back.test_func.Assign(PackComparisonOp(regs.stencil_back_func_func));
+    } else {
+        back.action_stencil_fail.Assign(front.action_stencil_fail);
+        back.action_depth_fail.Assign(front.action_depth_fail);
+        back.action_depth_pass.Assign(front.action_depth_pass);
+        back.test_func.Assign(front.test_func);
+    }
+    depth_test_enable.Assign(regs.depth_test_enable);
+    depth_write_enable.Assign(regs.depth_write_enabled);
+    depth_bounds_enable.Assign(regs.depth_bounds_enable);
+    stencil_enable.Assign(regs.stencil_enable);
+    depth_test_func.Assign(PackComparisonOp(regs.depth_test_func));
 }
 
+namespace {
+
 constexpr FixedPipelineState::InputAssembly GetInputAssemblyState(const Maxwell& regs) {
     return FixedPipelineState::InputAssembly(
         regs.draw.topology, regs.primitive_restart.enabled,
@@ -129,19 +138,6 @@ constexpr FixedPipelineState::Rasterizer GetRasterizerState(const Maxwell& regs)
 
 } // Anonymous namespace
 
-std::size_t FixedPipelineState::StencilFace::Hash() const noexcept {
-    return static_cast<std::size_t>(action_stencil_fail) ^
-           (static_cast<std::size_t>(action_depth_fail) << 4) ^
-           (static_cast<std::size_t>(action_depth_fail) << 20) ^
-           (static_cast<std::size_t>(action_depth_pass) << 36);
-}
-
-bool FixedPipelineState::StencilFace::operator==(const StencilFace& rhs) const noexcept {
-    return std::tie(action_stencil_fail, action_depth_fail, action_depth_pass, test_func) ==
-           std::tie(rhs.action_stencil_fail, rhs.action_depth_fail, rhs.action_depth_pass,
-                    rhs.test_func);
-}
-
 std::size_t FixedPipelineState::BlendingAttachment::Hash() const noexcept {
     return static_cast<std::size_t>(enable) ^ (static_cast<std::size_t>(rgb_equation) << 5) ^
            (static_cast<std::size_t>(src_rgb_func) << 10) ^
@@ -212,22 +208,11 @@ bool FixedPipelineState::Rasterizer::operator==(const Rasterizer& rhs) const noe
 }
 
 std::size_t FixedPipelineState::DepthStencil::Hash() const noexcept {
-    std::size_t hash = static_cast<std::size_t>(depth_test_enable) ^
-                       (static_cast<std::size_t>(depth_write_enable) << 1) ^
-                       (static_cast<std::size_t>(depth_bounds_enable) << 2) ^
-                       (static_cast<std::size_t>(stencil_enable) << 3) ^
-                       (static_cast<std::size_t>(depth_test_function) << 4);
-    boost::hash_combine(hash, front_stencil.Hash());
-    boost::hash_combine(hash, back_stencil.Hash());
-    return hash;
+    return raw;
 }
 
 bool FixedPipelineState::DepthStencil::operator==(const DepthStencil& rhs) const noexcept {
-    return std::tie(depth_test_enable, depth_write_enable, depth_bounds_enable, depth_test_function,
-                    stencil_enable, front_stencil, back_stencil) ==
-           std::tie(rhs.depth_test_enable, rhs.depth_write_enable, rhs.depth_bounds_enable,
-                    rhs.depth_test_function, rhs.stencil_enable, rhs.front_stencil,
-                    rhs.back_stencil);
+    return raw == rhs.raw;
 }
 
 std::size_t FixedPipelineState::ColorBlending::Hash() const noexcept {
@@ -266,9 +251,60 @@ FixedPipelineState GetFixedPipelineState(const Maxwell& regs) {
     fixed_state.input_assembly = GetInputAssemblyState(regs);
     fixed_state.tessellation = GetTessellationState(regs);
     fixed_state.rasterizer = GetRasterizerState(regs);
-    fixed_state.depth_stencil = GetDepthStencilState(regs);
+    fixed_state.depth_stencil.Fill(regs);
     fixed_state.color_blending = GetColorBlendingState(regs);
     return fixed_state;
 }
 
+u32 FixedPipelineState::PackComparisonOp(Maxwell::ComparisonOp op) noexcept {
+    // OpenGL enums go from 0x200 to 0x207 and the others from 1 to 8
+    // If we substract 0x200 to OpenGL enums and 1 to the others we get a 0-7 range.
+    // Perfect for a hash.
+    const u32 value = static_cast<u32>(op);
+    return value - (value >= 0x200 ? 0x200 : 1);
+}
+
+Maxwell::ComparisonOp FixedPipelineState::UnpackComparisonOp(u32 packed) noexcept {
+    // Read PackComparisonOp for the logic behind this.
+    return static_cast<Maxwell::ComparisonOp>(packed + 1);
+}
+
+u32 FixedPipelineState::PackStencilOp(Maxwell::StencilOp op) noexcept {
+    switch (op) {
+    case Maxwell::StencilOp::Keep:
+    case Maxwell::StencilOp::KeepOGL:
+        return 0;
+    case Maxwell::StencilOp::Zero:
+    case Maxwell::StencilOp::ZeroOGL:
+        return 1;
+    case Maxwell::StencilOp::Replace:
+    case Maxwell::StencilOp::ReplaceOGL:
+        return 2;
+    case Maxwell::StencilOp::Incr:
+    case Maxwell::StencilOp::IncrOGL:
+        return 3;
+    case Maxwell::StencilOp::Decr:
+    case Maxwell::StencilOp::DecrOGL:
+        return 4;
+    case Maxwell::StencilOp::Invert:
+    case Maxwell::StencilOp::InvertOGL:
+        return 5;
+    case Maxwell::StencilOp::IncrWrap:
+    case Maxwell::StencilOp::IncrWrapOGL:
+        return 6;
+    case Maxwell::StencilOp::DecrWrap:
+    case Maxwell::StencilOp::DecrWrapOGL:
+        return 7;
+    }
+    return 0;
+}
+
+Maxwell::StencilOp FixedPipelineState::UnpackStencilOp(u32 packed) noexcept {
+    static constexpr std::array LUT = {Maxwell::StencilOp::Keep,     Maxwell::StencilOp::Zero,
+                                       Maxwell::StencilOp::Replace,  Maxwell::StencilOp::Incr,
+                                       Maxwell::StencilOp::Decr,     Maxwell::StencilOp::Invert,
+                                       Maxwell::StencilOp::IncrWrap, Maxwell::StencilOp::DecrWrap};
+    return LUT[packed];
+}
+
 } // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.h b/src/video_core/renderer_vulkan/fixed_pipeline_state.h
index d82a82f75..e30877e77 100644
--- a/src/video_core/renderer_vulkan/fixed_pipeline_state.h
+++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.h
@@ -24,27 +24,11 @@ inline constexpr bool IsHashable = std::has_unique_object_representations_v<T>&&
     std::is_trivially_copyable_v<T>&& std::is_trivially_constructible_v<T>;
 
 struct FixedPipelineState {
-    struct StencilFace {
-        constexpr StencilFace(Maxwell::StencilOp action_stencil_fail,
-                              Maxwell::StencilOp action_depth_fail,
-                              Maxwell::StencilOp action_depth_pass, Maxwell::ComparisonOp test_func)
-            : action_stencil_fail{action_stencil_fail}, action_depth_fail{action_depth_fail},
-              action_depth_pass{action_depth_pass}, test_func{test_func} {}
-        StencilFace() = default;
-
-        Maxwell::StencilOp action_stencil_fail;
-        Maxwell::StencilOp action_depth_fail;
-        Maxwell::StencilOp action_depth_pass;
-        Maxwell::ComparisonOp test_func;
+    static u32 PackComparisonOp(Maxwell::ComparisonOp op) noexcept;
+    static Maxwell::ComparisonOp UnpackComparisonOp(u32 packed) noexcept;
 
-        std::size_t Hash() const noexcept;
-
-        bool operator==(const StencilFace& rhs) const noexcept;
-
-        bool operator!=(const StencilFace& rhs) const noexcept {
-            return !operator==(rhs);
-        }
-    };
+    static u32 PackStencilOp(Maxwell::StencilOp op) noexcept;
+    static Maxwell::StencilOp UnpackStencilOp(u32 packed) noexcept;
 
     struct BlendingAttachment {
         constexpr BlendingAttachment(bool enable, Maxwell::Blend::Equation rgb_equation,
@@ -202,23 +186,42 @@ struct FixedPipelineState {
     };
 
     struct DepthStencil {
-        constexpr DepthStencil(bool depth_test_enable, bool depth_write_enable,
-                               bool depth_bounds_enable, bool stencil_enable,
-                               Maxwell::ComparisonOp depth_test_function, StencilFace front_stencil,
-                               StencilFace back_stencil)
-            : depth_test_enable{depth_test_enable}, depth_write_enable{depth_write_enable},
-              depth_bounds_enable{depth_bounds_enable}, stencil_enable{stencil_enable},
-              depth_test_function{depth_test_function}, front_stencil{front_stencil},
-              back_stencil{back_stencil} {}
-        DepthStencil() = default;
-
-        bool depth_test_enable;
-        bool depth_write_enable;
-        bool depth_bounds_enable;
-        bool stencil_enable;
-        Maxwell::ComparisonOp depth_test_function;
-        StencilFace front_stencil;
-        StencilFace back_stencil;
+        template <std::size_t Position>
+        union StencilFace {
+            BitField<Position + 0, 3, u32> action_stencil_fail;
+            BitField<Position + 3, 3, u32> action_depth_fail;
+            BitField<Position + 6, 3, u32> action_depth_pass;
+            BitField<Position + 9, 3, u32> test_func;
+
+            Maxwell::StencilOp ActionStencilFail() const noexcept {
+                return UnpackStencilOp(action_stencil_fail);
+            }
+
+            Maxwell::StencilOp ActionDepthFail() const noexcept {
+                return UnpackStencilOp(action_depth_fail);
+            }
+
+            Maxwell::StencilOp ActionDepthPass() const noexcept {
+                return UnpackStencilOp(action_depth_pass);
+            }
+
+            Maxwell::ComparisonOp TestFunc() const noexcept {
+                return UnpackComparisonOp(test_func);
+            }
+        };
+
+        union {
+            u32 raw;
+            StencilFace<0> front;
+            StencilFace<12> back;
+            BitField<24, 1, u32> depth_test_enable;
+            BitField<25, 1, u32> depth_write_enable;
+            BitField<26, 1, u32> depth_bounds_enable;
+            BitField<27, 1, u32> stencil_enable;
+            BitField<28, 3, u32> depth_test_func;
+        };
+
+        void Fill(const Maxwell& regs) noexcept;
 
         std::size_t Hash() const noexcept;
 
@@ -227,7 +230,12 @@ struct FixedPipelineState {
         bool operator!=(const DepthStencil& rhs) const noexcept {
             return !operator==(rhs);
         }
+
+        Maxwell::ComparisonOp DepthTestFunc() const noexcept {
+            return UnpackComparisonOp(depth_test_func);
+        }
     };
+    static_assert(IsHashable<DepthStencil>);
 
     struct ColorBlending {
         constexpr ColorBlending(
@@ -248,6 +256,13 @@ struct FixedPipelineState {
         }
     };
 
+    VertexInput vertex_input;
+    InputAssembly input_assembly;
+    Tessellation tessellation;
+    Rasterizer rasterizer;
+    DepthStencil depth_stencil;
+    ColorBlending color_blending;
+
     std::size_t Hash() const noexcept;
 
     bool operator==(const FixedPipelineState& rhs) const noexcept;
@@ -255,15 +270,7 @@ struct FixedPipelineState {
     bool operator!=(const FixedPipelineState& rhs) const noexcept {
         return !operator==(rhs);
     }
-
-    VertexInput vertex_input;
-    InputAssembly input_assembly;
-    Tessellation tessellation;
-    Rasterizer rasterizer;
-    DepthStencil depth_stencil;
-    ColorBlending color_blending;
 };
-static_assert(std::is_trivially_copyable_v<FixedPipelineState::StencilFace>);
 static_assert(std::is_trivially_copyable_v<FixedPipelineState::BlendingAttachment>);
 static_assert(std::is_trivially_copyable_v<FixedPipelineState::VertexInput>);
 static_assert(std::is_trivially_copyable_v<FixedPipelineState::InputAssembly>);
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
index 718feafbd..0dd3ea5bc 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
@@ -26,12 +26,13 @@ MICROPROFILE_DECLARE(Vulkan_PipelineCache);
 
 namespace {
 
-VkStencilOpState GetStencilFaceState(const FixedPipelineState::StencilFace& face) {
+template <class StencilFace>
+VkStencilOpState GetStencilFaceState(const StencilFace& face) {
     VkStencilOpState state;
-    state.failOp = MaxwellToVK::StencilOp(face.action_stencil_fail);
-    state.passOp = MaxwellToVK::StencilOp(face.action_depth_pass);
-    state.depthFailOp = MaxwellToVK::StencilOp(face.action_depth_fail);
-    state.compareOp = MaxwellToVK::ComparisonOp(face.test_func);
+    state.failOp = MaxwellToVK::StencilOp(face.ActionStencilFail());
+    state.passOp = MaxwellToVK::StencilOp(face.ActionDepthPass());
+    state.depthFailOp = MaxwellToVK::StencilOp(face.ActionDepthFail());
+    state.compareOp = MaxwellToVK::ComparisonOp(face.TestFunc());
     state.compareMask = 0;
     state.writeMask = 0;
     state.reference = 0;
@@ -277,13 +278,12 @@ vk::Pipeline VKGraphicsPipeline::CreatePipeline(const RenderPassParams& renderpa
     depth_stencil_ci.flags = 0;
     depth_stencil_ci.depthTestEnable = ds.depth_test_enable;
     depth_stencil_ci.depthWriteEnable = ds.depth_write_enable;
-    depth_stencil_ci.depthCompareOp = ds.depth_test_enable
-                                          ? MaxwellToVK::ComparisonOp(ds.depth_test_function)
-                                          : VK_COMPARE_OP_ALWAYS;
+    depth_stencil_ci.depthCompareOp =
+        ds.depth_test_enable ? MaxwellToVK::ComparisonOp(ds.DepthTestFunc()) : VK_COMPARE_OP_ALWAYS;
     depth_stencil_ci.depthBoundsTestEnable = ds.depth_bounds_enable;
     depth_stencil_ci.stencilTestEnable = ds.stencil_enable;
-    depth_stencil_ci.front = GetStencilFaceState(ds.front_stencil);
-    depth_stencil_ci.back = GetStencilFaceState(ds.back_stencil);
+    depth_stencil_ci.front = GetStencilFaceState(ds.front);
+    depth_stencil_ci.back = GetStencilFaceState(ds.back);
     depth_stencil_ci.minDepthBounds = 0.0f;
     depth_stencil_ci.maxDepthBounds = 0.0f;