HostShaders: Fix D24S8 convertion shaders.

6 files changed, 47 insertions, 23 deletions

diff --git a/src/video_core/host_shaders/convert_abgr8_to_d24s8.frag b/src/video_core/host_shaders/convert_abgr8_to_d24s8.frag
index 4e4ab6a26..d51397a0c 100644
--- a/src/video_core/host_shaders/convert_abgr8_to_d24s8.frag
+++ b/src/video_core/host_shaders/convert_abgr8_to_d24s8.frag
@@ -10,8 +10,9 @@ layout(binding = 0) uniform sampler2D color_texture;
 void main() {
     ivec2 coord = ivec2(gl_FragCoord.xy);
     uvec4 color = uvec4(texelFetch(color_texture, coord, 0).rgba * (exp2(8) - 1.0f));
-    uint depth_unorm = (color.r << 16) | (color.g << 8) | color.b;
+    uvec4 bytes = color << uvec4(24, 16, 8, 0);
+    uint depth_stencil_unorm = bytes.x | bytes.y | bytes.z | bytes.w;
 
-    gl_FragDepth = float(depth_unorm) / (exp2(24.0) - 1.0f);
-    gl_FragStencilRefARB = int(color.a);
+    gl_FragDepth = float(depth_stencil_unorm & 0x00FFFFFFu) / (exp2(24.0) - 1.0f);
+    gl_FragStencilRefARB = int(depth_stencil_unorm >> 24);
 }
diff --git a/src/video_core/host_shaders/convert_b10g11r11_to_d24s8.frag b/src/video_core/host_shaders/convert_b10g11r11_to_d24s8.frag
index 2999a84cf..11bdd861d 100644
--- a/src/video_core/host_shaders/convert_b10g11r11_to_d24s8.frag
+++ b/src/video_core/host_shaders/convert_b10g11r11_to_d24s8.frag
@@ -7,13 +7,21 @@
 
 layout(binding = 0) uniform sampler2D color_texture;
 
+uint conv_from_float(float value_f, uint mantissa_bits) {
+    uint value = floatBitsToInt(value_f);
+    uint exp = (value >> 23) & 0x1Fu;
+    uint mantissa_shift = 32u - mantissa_bits;
+    uint mantissa = (value << 9u) >> mantissa_shift;
+    return (exp << mantissa_bits) | mantissa;
+}
+
 void main() {
     ivec2 coord = ivec2(gl_FragCoord.xy);
     vec4 color = texelFetch(color_texture, coord, 0).rgba;
-    uint depth_stencil_unorm = (uint(color.b * (exp2(10) - 1.0f)) << 22)
-                      | (uint(color.g * (exp2(11) - 1.0f)) << 11)
-                      | (uint(color.r * (exp2(11) - 1.0f)));
+    uint depth_stencil_unorm = (conv_from_float(color.r, 6u) << 21)
+                      | (conv_from_float(color.g, 6u) << 10)
+                      | conv_from_float(color.b, 5u);
 
-    gl_FragDepth = float(depth_stencil_unorm >> 8) / (exp2(24.0) - 1.0f);
-    gl_FragStencilRefARB = int(depth_stencil_unorm & 0x00FF);
+    gl_FragDepth = float(depth_stencil_unorm & 0x00FFFFFFu) / (exp2(24.0) - 1.0f);
+    gl_FragStencilRefARB = int(depth_stencil_unorm >> 24);
 }
diff --git a/src/video_core/host_shaders/convert_d24s8_to_abgr8.frag b/src/video_core/host_shaders/convert_d24s8_to_abgr8.frag
index ff3bf8209..47f9c1abc 100644
--- a/src/video_core/host_shaders/convert_d24s8_to_abgr8.frag
+++ b/src/video_core/host_shaders/convert_d24s8_to_abgr8.frag
@@ -14,8 +14,10 @@ void main() {
     uint depth = uint(textureLod(depth_tex, coord, 0).r * (exp2(24.0) - 1.0f));
     uint stencil = uint(textureLod(stencil_tex, coord, 0).r);
 
-    color.r = float(depth >> 16) / (exp2(8) - 1.0);
-    color.g = float((depth >> 8) & 0x00FF) / (exp2(8) - 1.0);
-    color.b = float(depth & 0x00FF) / (exp2(8) - 1.0);
-    color.a = float(stencil) / (exp2(8) - 1.0);
+    highp uint depth_val =
+        uint(textureLod(depth_tex, coord, 0).r * (exp2(32.0) - 1.0));
+    lowp uint stencil_val = textureLod(stencil_tex, coord, 0).r;
+    highp uvec4 components =
+        uvec4(stencil_val, (uvec3(depth_val) >> uvec3(24u, 16u, 8u)) & 0x000000FFu);
+    color = vec4(components) / (exp2(8.0) - 1.0);
 }
diff --git a/src/video_core/host_shaders/convert_d24s8_to_b10g11r11.frag b/src/video_core/host_shaders/convert_d24s8_to_b10g11r11.frag
index c743d3a13..c2d935fcd 100644
--- a/src/video_core/host_shaders/convert_d24s8_to_b10g11r11.frag
+++ b/src/video_core/host_shaders/convert_d24s8_to_b10g11r11.frag
@@ -9,13 +9,24 @@ layout(binding = 1) uniform isampler2D stencil_tex;
 
 layout(location = 0) out vec4 color;
 
+float conv_to_float(uint value, uint mantissa_bits) {
+    uint exp = (value >> mantissa_bits) & 0x1Fu;
+    uint mantissa_shift = 32u - mantissa_bits;
+    uint mantissa = (value << mantissa_shift) >> mantissa_shift;
+    return uintBitsToFloat((exp << 23) | (mantissa << (23 - mantissa_bits)));
+}
+
 void main() {
     ivec2 coord = ivec2(gl_FragCoord.xy);
-    uint depth = uint(textureLod(depth_tex, coord, 0).r * (exp2(24.0) - 1.0f));
+    uint depth = uint(textureLod(depth_tex, coord, 0).r * (exp2(32.0) - 1.0f));
     uint stencil = uint(textureLod(stencil_tex, coord, 0).r);
+    uint depth_stencil = (stencil << 24) | (depth >> 8);
+    uint red_int = (depth_stencil >> 21) & 0x07FF;
+    uint green_int = (depth_stencil >> 10) & 0x07FF;
+    uint blue_int = depth_stencil & 0x03FF;
 
-    color.b = float(depth >> 22) / (exp2(10) - 1.0);
-    color.g = float((depth >> 11) & 0x00FF) / (exp2(11) - 1.0);
-    color.r = float(depth & 0x00FF) / (exp2(11) - 1.0);
+    color.r = conv_to_float(red_int, 6u);
+    color.g = conv_to_float(green_int, 6u);
+    color.b = conv_to_float(blue_int, 5u);
     color.a = 1.0f;
 }
diff --git a/src/video_core/host_shaders/convert_d24s8_to_r16g16.frag b/src/video_core/host_shaders/convert_d24s8_to_r16g16.frag
index 2a9443d3d..c48a7ac66 100644
--- a/src/video_core/host_shaders/convert_d24s8_to_r16g16.frag
+++ b/src/video_core/host_shaders/convert_d24s8_to_r16g16.frag
@@ -11,11 +11,12 @@ layout(location = 0) out vec4 color;
 
 void main() {
     ivec2 coord = ivec2(gl_FragCoord.xy);
-    uint depth = uint(textureLod(depth_tex, coord, 0).r * (exp2(24.0) - 1.0f));
+    uint depth = uint(textureLod(depth_tex, coord, 0).r * (exp2(32.0) - 1.0f));
     uint stencil = uint(textureLod(stencil_tex, coord, 0).r);
+    uint depth_stencil = (stencil << 24) | (depth >> 8);
 
-    color.r = float(depth >> 16) / (exp2(16) - 1.0);
-    color.g = float((depth >> 16) & 0x00FF) / (exp2(16) - 1.0);
+    color.r = float(depth_stencil & 0x0000FFFFu) / (exp2(16) - 1.0);
+    color.g = float(depth_stencil >> 16) / (exp2(16) - 1.0);
     color.b = 0.0f;
     color.a = 1.0f;
 }
diff --git a/src/video_core/host_shaders/convert_r16g16_to_d24s8.frag b/src/video_core/host_shaders/convert_r16g16_to_d24s8.frag
index 3df70575e..beb2d1284 100644
--- a/src/video_core/host_shaders/convert_r16g16_to_d24s8.frag
+++ b/src/video_core/host_shaders/convert_r16g16_to_d24s8.frag
@@ -10,9 +10,10 @@ layout(binding = 0) uniform sampler2D color_texture;
 void main() {
     ivec2 coord = ivec2(gl_FragCoord.xy);
     vec4 color = texelFetch(color_texture, coord, 0).rgba;
-    uint depth_stencil_unorm = (uint(color.r * (exp2(16) - 1.0f)) << 16)
-                      | (uint(color.g * (exp2(16) - 1.0f)) << 16);
+    uvec2 bytes = uvec2(color.rg * (exp2(16) - 1.0f)) << uvec2(0, 16);
+    uint depth_stencil_unorm =
+        uint(color.r * (exp2(16) - 1.0f)) | (uint(color.g * (exp2(16) - 1.0f)) << 16);
 
-    gl_FragDepth = float(depth_stencil_unorm >> 8) / (exp2(24.0) - 1.0f);
-    gl_FragStencilRefARB = int(depth_stencil_unorm & 0x00FF);
+    gl_FragDepth = float(depth_stencil_unorm & 0x00FFFFFFu) / (exp2(24.0) - 1.0f);
+    gl_FragStencilRefARB = int(depth_stencil_unorm >> 24);
 }