Merge pull request #1153 from bunnei/stencil-clear

gl_rasterizer: Implement partial color clear, stencil clear, and stencil test.

6 files changed, 188 insertions, 69 deletions

diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index d03bc1c0c..92bfda053 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -330,6 +330,17 @@ public:
             Set = 0x150F,
         };
 
+        enum class StencilOp : u32 {
+            Keep = 1,
+            Zero = 2,
+            Replace = 3,
+            Incr = 4,
+            Decr = 5,
+            Invert = 6,
+            IncrWrap = 7,
+            DecrWrap = 8,
+        };
+
         struct Cull {
             enum class FrontFace : u32 {
                 ClockWise = 0x0900,
@@ -508,8 +519,16 @@ public:
 
                 float clear_color[4];
                 float clear_depth;
+                INSERT_PADDING_WORDS(0x3);
+                s32 clear_stencil;
+
+                INSERT_PADDING_WORDS(0x6C);
+
+                s32 stencil_back_func_ref;
+                u32 stencil_back_mask;
+                u32 stencil_back_func_mask;
 
-                INSERT_PADDING_WORDS(0x93);
+                INSERT_PADDING_WORDS(0x20);
 
                 struct {
                     u32 address_high;
@@ -573,16 +592,14 @@ public:
                     u32 enable[NumRenderTargets];
                 } blend;
 
-                struct {
-                    u32 enable;
-                    u32 front_op_fail;
-                    u32 front_op_zfail;
-                    u32 front_op_zpass;
-                    u32 front_func_func;
-                    u32 front_func_ref;
-                    u32 front_func_mask;
-                    u32 front_mask;
-                } stencil;
+                u32 stencil_enable;
+                StencilOp stencil_front_op_fail;
+                StencilOp stencil_front_op_zfail;
+                StencilOp stencil_front_op_zpass;
+                ComparisonOp stencil_front_func_func;
+                s32 stencil_front_func_ref;
+                u32 stencil_front_func_mask;
+                u32 stencil_front_mask;
 
                 INSERT_PADDING_WORDS(0x3);
 
@@ -626,13 +643,11 @@ public:
 
                 INSERT_PADDING_WORDS(0x5);
 
-                struct {
-                    u32 enable;
-                    u32 back_op_fail;
-                    u32 back_op_zfail;
-                    u32 back_op_zpass;
-                    u32 back_func_func;
-                } stencil_two_side;
+                u32 stencil_two_side_enable;
+                StencilOp stencil_back_op_fail;
+                StencilOp stencil_back_op_zfail;
+                StencilOp stencil_back_op_zpass;
+                ComparisonOp stencil_back_func_func;
 
                 INSERT_PADDING_WORDS(0x17);
 
@@ -944,6 +959,10 @@ ASSERT_REG_POSITION(viewport, 0x300);
 ASSERT_REG_POSITION(vertex_buffer, 0x35D);
 ASSERT_REG_POSITION(clear_color[0], 0x360);
 ASSERT_REG_POSITION(clear_depth, 0x364);
+ASSERT_REG_POSITION(clear_stencil, 0x368);
+ASSERT_REG_POSITION(stencil_back_func_ref, 0x3D5);
+ASSERT_REG_POSITION(stencil_back_mask, 0x3D6);
+ASSERT_REG_POSITION(stencil_back_func_mask, 0x3D7);
 ASSERT_REG_POSITION(zeta, 0x3F8);
 ASSERT_REG_POSITION(vertex_attrib_format[0], 0x458);
 ASSERT_REG_POSITION(rt_control, 0x487);
@@ -955,13 +974,24 @@ ASSERT_REG_POSITION(depth_write_enabled, 0x4BA);
 ASSERT_REG_POSITION(d3d_cull_mode, 0x4C2);
 ASSERT_REG_POSITION(depth_test_func, 0x4C3);
 ASSERT_REG_POSITION(blend, 0x4CF);
-ASSERT_REG_POSITION(stencil, 0x4E0);
+ASSERT_REG_POSITION(stencil_enable, 0x4E0);
+ASSERT_REG_POSITION(stencil_front_op_fail, 0x4E1);
+ASSERT_REG_POSITION(stencil_front_op_zfail, 0x4E2);
+ASSERT_REG_POSITION(stencil_front_op_zpass, 0x4E3);
+ASSERT_REG_POSITION(stencil_front_func_func, 0x4E4);
+ASSERT_REG_POSITION(stencil_front_func_ref, 0x4E5);
+ASSERT_REG_POSITION(stencil_front_func_mask, 0x4E6);
+ASSERT_REG_POSITION(stencil_front_mask, 0x4E7);
 ASSERT_REG_POSITION(screen_y_control, 0x4EB);
 ASSERT_REG_POSITION(vb_element_base, 0x50D);
 ASSERT_REG_POSITION(zeta_enable, 0x54E);
 ASSERT_REG_POSITION(tsc, 0x557);
 ASSERT_REG_POSITION(tic, 0x55D);
-ASSERT_REG_POSITION(stencil_two_side, 0x565);
+ASSERT_REG_POSITION(stencil_two_side_enable, 0x565);
+ASSERT_REG_POSITION(stencil_back_op_fail, 0x566);
+ASSERT_REG_POSITION(stencil_back_op_zfail, 0x567);
+ASSERT_REG_POSITION(stencil_back_op_zpass, 0x568);
+ASSERT_REG_POSITION(stencil_back_func_func, 0x569);
 ASSERT_REG_POSITION(point_coord_replace, 0x581);
 ASSERT_REG_POSITION(code_address, 0x582);
 ASSERT_REG_POSITION(draw, 0x585);
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 516e1b50f..8bfa75b84 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -14,6 +14,7 @@
 #include "common/logging/log.h"
 #include "common/math_util.h"
 #include "common/microprofile.h"
+#include "common/scope_exit.h"
 #include "core/core.h"
 #include "core/frontend/emu_window.h"
 #include "core/hle/kernel/process.h"
@@ -315,16 +316,14 @@ std::pair<Surface, Surface> RasterizerOpenGL::ConfigureFramebuffers(bool using_c
         using_color_fb = false;
     }
 
-    // TODO(bunnei): Implement this
-    const bool has_stencil = false;
-
+    const bool has_stencil = regs.stencil_enable;
     const bool write_color_fb =
         state.color_mask.red_enabled == GL_TRUE || state.color_mask.green_enabled == GL_TRUE ||
         state.color_mask.blue_enabled == GL_TRUE || state.color_mask.alpha_enabled == GL_TRUE;
 
     const bool write_depth_fb =
         (state.depth.test_enabled && state.depth.write_mask == GL_TRUE) ||
-        (has_stencil && state.stencil.test_enabled && state.stencil.write_mask != 0);
+        (has_stencil && (state.stencil.front.write_mask || state.stencil.back.write_mask));
 
     Surface color_surface;
     Surface depth_surface;
@@ -364,41 +363,70 @@ std::pair<Surface, Surface> RasterizerOpenGL::ConfigureFramebuffers(bool using_c
 }
 
 void RasterizerOpenGL::Clear() {
-    const auto& regs = Core::System::GetInstance().GPU().Maxwell3D().regs;
+    const auto prev_state{state};
+    SCOPE_EXIT({ prev_state.Apply(); });
 
+    const auto& regs = Core::System::GetInstance().GPU().Maxwell3D().regs;
     bool use_color_fb = false;
     bool use_depth_fb = false;
 
-    GLbitfield clear_mask = 0;
-    if (regs.clear_buffers.R && regs.clear_buffers.G && regs.clear_buffers.B &&
+    OpenGLState clear_state;
+    clear_state.draw.draw_framebuffer = state.draw.draw_framebuffer;
+    clear_state.color_mask.red_enabled = regs.clear_buffers.R ? GL_TRUE : GL_FALSE;
+    clear_state.color_mask.green_enabled = regs.clear_buffers.G ? GL_TRUE : GL_FALSE;
+    clear_state.color_mask.blue_enabled = regs.clear_buffers.B ? GL_TRUE : GL_FALSE;
+    clear_state.color_mask.alpha_enabled = regs.clear_buffers.A ? GL_TRUE : GL_FALSE;
+
+    GLbitfield clear_mask{};
+    if (regs.clear_buffers.R || regs.clear_buffers.G || regs.clear_buffers.B ||
         regs.clear_buffers.A) {
-        clear_mask |= GL_COLOR_BUFFER_BIT;
-        use_color_fb = true;
+        if (regs.clear_buffers.RT == 0) {
+            // We only support clearing the first color attachment for now
+            clear_mask |= GL_COLOR_BUFFER_BIT;
+            use_color_fb = true;
+        } else {
+            // TODO(subv): Add support for the other color attachments
+            LOG_CRITICAL(HW_GPU, "Clear unimplemented for RT {}", regs.clear_buffers.RT);
+        }
     }
     if (regs.clear_buffers.Z) {
+        ASSERT_MSG(regs.zeta_enable != 0, "Tried to clear Z but buffer is not enabled!");
+        use_depth_fb = true;
         clear_mask |= GL_DEPTH_BUFFER_BIT;
-        use_depth_fb = regs.zeta_enable != 0;
 
         // Always enable the depth write when clearing the depth buffer. The depth write mask is
         // ignored when clearing the buffer in the Switch, but OpenGL obeys it so we set it to true.
-        state.depth.test_enabled = true;
-        state.depth.write_mask = GL_TRUE;
-        state.depth.test_func = GL_ALWAYS;
-        state.Apply();
+        clear_state.depth.test_enabled = true;
+        clear_state.depth.test_func = GL_ALWAYS;
+    }
+    if (regs.clear_buffers.S) {
+        ASSERT_MSG(regs.zeta_enable != 0, "Tried to clear stencil but buffer is not enabled!");
+        use_depth_fb = true;
+        clear_mask |= GL_STENCIL_BUFFER_BIT;
+        clear_state.stencil.test_enabled = true;
     }
 
-    if (clear_mask == 0)
+    if (!use_color_fb && !use_depth_fb) {
+        // No color surface nor depth/stencil surface are enabled
         return;
+    }
+
+    if (clear_mask == 0) {
+        // No clear mask is enabled
+        return;
+    }
 
     ScopeAcquireGLContext acquire_context{emu_window};
 
     auto [dirty_color_surface, dirty_depth_surface] =
         ConfigureFramebuffers(use_color_fb, use_depth_fb, false);
 
-    // TODO(Subv): Support clearing only partial colors.
+    clear_state.Apply();
+
     glClearColor(regs.clear_color[0], regs.clear_color[1], regs.clear_color[2],
                  regs.clear_color[3]);
     glClearDepth(regs.clear_depth);
+    glClearStencil(regs.clear_stencil);
 
     glClear(clear_mask);
 
@@ -451,6 +479,7 @@ void RasterizerOpenGL::DrawArrays() {
         ConfigureFramebuffers(true, regs.zeta.Address() != 0 && regs.zeta_enable != 0, true);
 
     SyncDepthTestState();
+    SyncStencilTestState();
     SyncBlendState();
     SyncLogicOpState();
     SyncCullMode();
@@ -841,6 +870,34 @@ void RasterizerOpenGL::SyncDepthTestState() {
     state.depth.test_func = MaxwellToGL::ComparisonOp(regs.depth_test_func);
 }
 
+void RasterizerOpenGL::SyncStencilTestState() {
+    const auto& regs = Core::System::GetInstance().GPU().Maxwell3D().regs;
+    state.stencil.test_enabled = regs.stencil_enable != 0;
+
+    if (!regs.stencil_enable) {
+        return;
+    }
+
+    // TODO(bunnei): Verify behavior when this is not set
+    ASSERT(regs.stencil_two_side_enable);
+
+    state.stencil.front.test_func = MaxwellToGL::ComparisonOp(regs.stencil_front_func_func);
+    state.stencil.front.test_ref = regs.stencil_front_func_ref;
+    state.stencil.front.test_mask = regs.stencil_front_func_mask;
+    state.stencil.front.action_stencil_fail = MaxwellToGL::StencilOp(regs.stencil_front_op_fail);
+    state.stencil.front.action_depth_fail = MaxwellToGL::StencilOp(regs.stencil_front_op_zfail);
+    state.stencil.front.action_depth_pass = MaxwellToGL::StencilOp(regs.stencil_front_op_zpass);
+    state.stencil.front.write_mask = regs.stencil_front_mask;
+
+    state.stencil.back.test_func = MaxwellToGL::ComparisonOp(regs.stencil_back_func_func);
+    state.stencil.back.test_ref = regs.stencil_back_func_ref;
+    state.stencil.back.test_mask = regs.stencil_back_func_mask;
+    state.stencil.back.action_stencil_fail = MaxwellToGL::StencilOp(regs.stencil_back_op_fail);
+    state.stencil.back.action_depth_fail = MaxwellToGL::StencilOp(regs.stencil_back_op_zfail);
+    state.stencil.back.action_depth_pass = MaxwellToGL::StencilOp(regs.stencil_back_op_zpass);
+    state.stencil.back.write_mask = regs.stencil_back_mask;
+}
+
 void RasterizerOpenGL::SyncBlendState() {
     const auto& regs = Core::System::GetInstance().GPU().Maxwell3D().regs;
 
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h
index 59b727de0..531b04046 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer.h
@@ -141,6 +141,9 @@ private:
     /// Syncs the depth test state to match the guest state
     void SyncDepthTestState();
 
+    /// Syncs the stencil test state to match the guest state
+    void SyncStencilTestState();
+
     /// Syncs the blend state to match the guest state
     void SyncBlendState();
 
diff --git a/src/video_core/renderer_opengl/gl_state.cpp b/src/video_core/renderer_opengl/gl_state.cpp
index e1a887d67..60a4defd1 100644
--- a/src/video_core/renderer_opengl/gl_state.cpp
+++ b/src/video_core/renderer_opengl/gl_state.cpp
@@ -27,13 +27,17 @@ OpenGLState::OpenGLState() {
     color_mask.alpha_enabled = GL_TRUE;
 
     stencil.test_enabled = false;
-    stencil.test_func = GL_ALWAYS;
-    stencil.test_ref = 0;
-    stencil.test_mask = 0xFF;
-    stencil.write_mask = 0xFF;
-    stencil.action_depth_fail = GL_KEEP;
-    stencil.action_depth_pass = GL_KEEP;
-    stencil.action_stencil_fail = GL_KEEP;
+    auto reset_stencil = [](auto& config) {
+        config.test_func = GL_ALWAYS;
+        config.test_ref = 0;
+        config.test_mask = 0xFFFFFFFF;
+        config.write_mask = 0xFFFFFFFF;
+        config.action_depth_fail = GL_KEEP;
+        config.action_depth_pass = GL_KEEP;
+        config.action_stencil_fail = GL_KEEP;
+    };
+    reset_stencil(stencil.front);
+    reset_stencil(stencil.back);
 
     blend.enabled = true;
     blend.rgb_equation = GL_FUNC_ADD;
@@ -129,24 +133,23 @@ void OpenGLState::Apply() const {
             glDisable(GL_STENCIL_TEST);
         }
     }
-
-    if (stencil.test_func != cur_state.stencil.test_func ||
-        stencil.test_ref != cur_state.stencil.test_ref ||
-        stencil.test_mask != cur_state.stencil.test_mask) {
-        glStencilFunc(stencil.test_func, stencil.test_ref, stencil.test_mask);
-    }
-
-    if (stencil.action_depth_fail != cur_state.stencil.action_depth_fail ||
-        stencil.action_depth_pass != cur_state.stencil.action_depth_pass ||
-        stencil.action_stencil_fail != cur_state.stencil.action_stencil_fail) {
-        glStencilOp(stencil.action_stencil_fail, stencil.action_depth_fail,
-                    stencil.action_depth_pass);
-    }
-
-    // Stencil mask
-    if (stencil.write_mask != cur_state.stencil.write_mask) {
-        glStencilMask(stencil.write_mask);
-    }
+    auto config_stencil = [](GLenum face, const auto& config, const auto& prev_config) {
+        if (config.test_func != prev_config.test_func || config.test_ref != prev_config.test_ref ||
+            config.test_mask != prev_config.test_mask) {
+            glStencilFuncSeparate(face, config.test_func, config.test_ref, config.test_mask);
+        }
+        if (config.action_depth_fail != prev_config.action_depth_fail ||
+            config.action_depth_pass != prev_config.action_depth_pass ||
+            config.action_stencil_fail != prev_config.action_stencil_fail) {
+            glStencilOpSeparate(face, config.action_stencil_fail, config.action_depth_fail,
+                                config.action_depth_pass);
+        }
+        if (config.write_mask != prev_config.write_mask) {
+            glStencilMaskSeparate(face, config.write_mask);
+        }
+    };
+    config_stencil(GL_FRONT, stencil.front, cur_state.stencil.front);
+    config_stencil(GL_BACK, stencil.back, cur_state.stencil.back);
 
     // Blending
     if (blend.enabled != cur_state.blend.enabled) {
diff --git a/src/video_core/renderer_opengl/gl_state.h b/src/video_core/renderer_opengl/gl_state.h
index 22b0b1e41..46e96a97d 100644
--- a/src/video_core/renderer_opengl/gl_state.h
+++ b/src/video_core/renderer_opengl/gl_state.h
@@ -58,14 +58,16 @@ public:
     } color_mask; // GL_COLOR_WRITEMASK
 
     struct {
-        bool test_enabled;          // GL_STENCIL_TEST
-        GLenum test_func;           // GL_STENCIL_FUNC
-        GLint test_ref;             // GL_STENCIL_REF
-        GLuint test_mask;           // GL_STENCIL_VALUE_MASK
-        GLuint write_mask;          // GL_STENCIL_WRITEMASK
-        GLenum action_stencil_fail; // GL_STENCIL_FAIL
-        GLenum action_depth_fail;   // GL_STENCIL_PASS_DEPTH_FAIL
-        GLenum action_depth_pass;   // GL_STENCIL_PASS_DEPTH_PASS
+        bool test_enabled; // GL_STENCIL_TEST
+        struct {
+            GLenum test_func;           // GL_STENCIL_FUNC
+            GLint test_ref;             // GL_STENCIL_REF
+            GLuint test_mask;           // GL_STENCIL_VALUE_MASK
+            GLuint write_mask;          // GL_STENCIL_WRITEMASK
+            GLenum action_stencil_fail; // GL_STENCIL_FAIL
+            GLenum action_depth_fail;   // GL_STENCIL_PASS_DEPTH_FAIL
+            GLenum action_depth_pass;   // GL_STENCIL_PASS_DEPTH_PASS
+        } front, back;
     } stencil;
 
     struct {
diff --git a/src/video_core/renderer_opengl/maxwell_to_gl.h b/src/video_core/renderer_opengl/maxwell_to_gl.h
index 0343759a6..67273e164 100644
--- a/src/video_core/renderer_opengl/maxwell_to_gl.h
+++ b/src/video_core/renderer_opengl/maxwell_to_gl.h
@@ -295,6 +295,30 @@ inline GLenum ComparisonOp(Maxwell::ComparisonOp comparison) {
     return {};
 }
 
+inline GLenum StencilOp(Maxwell::StencilOp stencil) {
+    switch (stencil) {
+    case Maxwell::StencilOp::Keep:
+        return GL_KEEP;
+    case Maxwell::StencilOp::Zero:
+        return GL_ZERO;
+    case Maxwell::StencilOp::Replace:
+        return GL_REPLACE;
+    case Maxwell::StencilOp::Incr:
+        return GL_INCR;
+    case Maxwell::StencilOp::Decr:
+        return GL_DECR;
+    case Maxwell::StencilOp::Invert:
+        return GL_INVERT;
+    case Maxwell::StencilOp::IncrWrap:
+        return GL_INCR_WRAP;
+    case Maxwell::StencilOp::DecrWrap:
+        return GL_DECR_WRAP;
+    }
+    LOG_CRITICAL(Render_OpenGL, "Unimplemented stencil op={}", static_cast<u32>(stencil));
+    UNREACHABLE();
+    return {};
+}
+
 inline GLenum FrontFace(Maxwell::Cull::FrontFace front_face) {
     switch (front_face) {
     case Maxwell::Cull::FrontFace::ClockWise: