shader: Fix F2I

6 files changed, 124 insertions, 5 deletions

diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
index ce610799a..6280c08f6 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
@@ -731,6 +731,24 @@ F16F32F64 IREmitter::FPSaturate(const F16F32F64& value) {
     }
 }
 
+F16F32F64 IREmitter::FPClamp(const F16F32F64& value, const F16F32F64& min_value,
+                             const F16F32F64& max_value) {
+    if (value.Type() != min_value.Type() || value.Type() != max_value.Type()) {
+        throw InvalidArgument("Mismatching types {}, {}, and {}", value.Type(), min_value.Type(),
+                              max_value.Type());
+    }
+    switch (value.Type()) {
+    case Type::F16:
+        return Inst<F16>(Opcode::FPClamp16, value, min_value, max_value);
+    case Type::F32:
+        return Inst<F32>(Opcode::FPClamp32, value, min_value, max_value);
+    case Type::F64:
+        return Inst<F64>(Opcode::FPClamp64, value, min_value, max_value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
 F16F32F64 IREmitter::FPRoundEven(const F16F32F64& value, FpControl control) {
     switch (value.Type()) {
     case Type::F16:
diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h
index 39109b0de..ebbda78a9 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.h
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.h
@@ -147,6 +147,7 @@ public:
     [[nodiscard]] F32F64 FPRecipSqrt(const F32F64& value);
     [[nodiscard]] F32 FPSqrt(const F32& value);
     [[nodiscard]] F16F32F64 FPSaturate(const F16F32F64& value);
+    [[nodiscard]] F16F32F64 FPClamp(const F16F32F64& value, const F16F32F64& min_value, const F16F32F64& max_value);
     [[nodiscard]] F16F32F64 FPRoundEven(const F16F32F64& value, FpControl control = {});
     [[nodiscard]] F16F32F64 FPFloor(const F16F32F64& value, FpControl control = {});
     [[nodiscard]] F16F32F64 FPCeil(const F16F32F64& value, FpControl control = {});
diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc
index 8945c7b04..dd17212a1 100644
--- a/src/shader_recompiler/frontend/ir/opcodes.inc
+++ b/src/shader_recompiler/frontend/ir/opcodes.inc
@@ -192,6 +192,9 @@ OPCODE(FPLog2,                                              F32,            F32,
 OPCODE(FPSaturate16,                                        F16,            F16,                                                                            )
 OPCODE(FPSaturate32,                                        F32,            F32,                                                                            )
 OPCODE(FPSaturate64,                                        F64,            F64,                                                                            )
+OPCODE(FPClamp16,                                           F16,            F16,            F16,            F16,                                            )
+OPCODE(FPClamp32,                                           F32,            F32,            F32,            F32,                                            )
+OPCODE(FPClamp64,                                           F64,            F64,            F64,            F64,                                            )
 OPCODE(FPRoundEven16,                                       F16,            F16,                                                                            )
 OPCODE(FPRoundEven32,                                       F32,            F32,                                                                            )
 OPCODE(FPRoundEven64,                                       F64,            F64,                                                                            )
diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp
index 81175627f..7c5a72800 100644
--- a/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp
+++ b/src/shader_recompiler/frontend/maxwell/translate/impl/floating_point_conversion_integer.cpp
@@ -2,6 +2,8 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <limits>
+
 #include "common/common_types.h"
 #include "shader_recompiler/exception.h"
 #include "shader_recompiler/frontend/maxwell/opcodes.h"
@@ -55,6 +57,37 @@ size_t BitSize(DestFormat dest_format) {
     }
 }
 
+std::pair<f64, f64> ClampBounds(DestFormat format, bool is_signed) {
+    if (is_signed) {
+        switch (format) {
+        case DestFormat::I16:
+            return {static_cast<f64>(std::numeric_limits<s16>::max()),
+                    static_cast<f64>(std::numeric_limits<s16>::min())};
+        case DestFormat::I32:
+            return {static_cast<f64>(std::numeric_limits<s32>::max()),
+                    static_cast<f64>(std::numeric_limits<s32>::min())};
+        case DestFormat::I64:
+            return {static_cast<f64>(std::numeric_limits<s64>::max()),
+                    static_cast<f64>(std::numeric_limits<s64>::min())};
+        default: {}
+        }
+    } else {
+        switch (format) {
+        case DestFormat::I16:
+            return {static_cast<f64>(std::numeric_limits<u16>::max()),
+                    static_cast<f64>(std::numeric_limits<u16>::min())};
+        case DestFormat::I32:
+            return {static_cast<f64>(std::numeric_limits<u32>::max()),
+                    static_cast<f64>(std::numeric_limits<u32>::min())};
+        case DestFormat::I64:
+            return {static_cast<f64>(std::numeric_limits<u64>::max()),
+                    static_cast<f64>(std::numeric_limits<u64>::min())};
+        default: {}
+        }
+    }
+    throw NotImplementedException("Invalid destination format {}", format);
+}
+
 IR::F64 UnpackCbuf(TranslatorVisitor& v, u64 insn) {
     union {
         u64 raw;
@@ -112,13 +145,58 @@ void TranslateF2I(TranslatorVisitor& v, u64 insn, const IR::F16F32F64& src_a) {
     // For example converting F32 65537.0 to U16, the expected value is 0xffff,
 
     const bool is_signed{f2i.is_signed != 0};
-    const size_t bitsize{BitSize(f2i.dest_format)};
-    const IR::U16U32U64 result{v.ir.ConvertFToI(bitsize, is_signed, rounded_value)};
+    const auto [max_bound, min_bound] = ClampBounds(f2i.dest_format, is_signed);
+
+    IR::F16F32F64 intermediate;
+    switch (f2i.src_format) {
+    case SrcFormat::F16: {
+        const IR::F16 max_val{v.ir.FPConvert(16, v.ir.Imm32(static_cast<f32>(max_bound)))};
+        const IR::F16 min_val{v.ir.FPConvert(16, v.ir.Imm32(static_cast<f32>(min_bound)))};
+        intermediate = v.ir.FPClamp(rounded_value, min_val, max_val);
+        break;
+    }
+    case SrcFormat::F32: {
+        const IR::F32 max_val{v.ir.Imm32(static_cast<f32>(max_bound))};
+        const IR::F32 min_val{v.ir.Imm32(static_cast<f32>(min_bound))};
+        intermediate = v.ir.FPClamp(rounded_value, min_val, max_val);
+        break;
+    }
+    case SrcFormat::F64: {
+        const IR::F64 max_val{v.ir.Imm64(max_bound)};
+        const IR::F64 min_val{v.ir.Imm64(min_bound)};
+        intermediate = v.ir.FPClamp(rounded_value, min_val, max_val);
+        break;
+    }
+    default:
+        throw NotImplementedException("Invalid destination format {}", f2i.dest_format.Value());
+    }
+
+    const size_t bitsize{std::max<size_t>(32, BitSize(f2i.dest_format))};
+    IR::U16U32U64 result{v.ir.ConvertFToI(bitsize, is_signed, intermediate)};
+
+    bool handled_special_case = false;
+    const bool special_nan_cases =
+        (f2i.src_format == SrcFormat::F64) != (f2i.dest_format == DestFormat::I64);
+    if (special_nan_cases) {
+        if (f2i.dest_format == DestFormat::I32) {
+            handled_special_case = true;
+            result = IR::U32{v.ir.Select(v.ir.FPIsNan(op_a), v.ir.Imm32(0x8000'0000U), result)};
+        } else if (f2i.dest_format == DestFormat::I64) {
+            handled_special_case = true;
+            result = IR::U64{
+                v.ir.Select(v.ir.FPIsNan(op_a), v.ir.Imm64(0x8000'0000'0000'0000ULL), result)};
+        }
+    }
+    if (!handled_special_case && is_signed) {
+        if (bitsize != 64) {
+            result = IR::U32{v.ir.Select(v.ir.FPIsNan(op_a), v.ir.Imm32(0U), result)};
+        } else {
+            result = IR::U64{v.ir.Select(v.ir.FPIsNan(op_a), v.ir.Imm64(0ULL), result)};
+        }
+    }
 
     if (bitsize == 64) {
-        const IR::Value vector{v.ir.UnpackUint2x32(result)};
-        v.X(f2i.dest_reg + 0, IR::U32{v.ir.CompositeExtract(vector, 0)});
-        v.X(f2i.dest_reg + 1, IR::U32{v.ir.CompositeExtract(vector, 1)});
+        v.L(f2i.dest_reg, result);
     } else {
         v.X(f2i.dest_reg, result);
     }
diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/impl.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/impl.cpp
index 758a0230a..9bae89c10 100644
--- a/src/shader_recompiler/frontend/maxwell/translate/impl/impl.cpp
+++ b/src/shader_recompiler/frontend/maxwell/translate/impl/impl.cpp
@@ -21,6 +21,13 @@ IR::U32 TranslatorVisitor::X(IR::Reg reg) {
     return ir.GetReg(reg);
 }
 
+IR::U64 TranslatorVisitor::L(IR::Reg reg) {
+    if (!IR::IsAligned(reg, 2)) {
+        throw NotImplementedException("Unaligned source register {}", reg);
+    }
+    return IR::U64{ir.PackUint2x32(ir.CompositeConstruct(X(reg), X(reg + 1)))};
+}
+
 IR::F32 TranslatorVisitor::F(IR::Reg reg) {
     return ir.BitCast<IR::F32>(X(reg));
 }
@@ -36,6 +43,16 @@ void TranslatorVisitor::X(IR::Reg dest_reg, const IR::U32& value) {
     ir.SetReg(dest_reg, value);
 }
 
+void TranslatorVisitor::L(IR::Reg dest_reg, const IR::U64& value) {
+    if (!IR::IsAligned(dest_reg, 2)) {
+        throw NotImplementedException("Unaligned destination register {}", dest_reg);
+    }
+    const IR::Value result{ir.UnpackUint2x32(value)};
+    for (int i = 0; i < 2; i++) {
+        X(dest_reg + i, IR::U32{ir.CompositeExtract(result, i)});
+    }
+}
+
 void TranslatorVisitor::F(IR::Reg dest_reg, const IR::F32& value) {
     X(dest_reg, ir.BitCast<IR::U32>(value));
 }
diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/impl.h b/src/shader_recompiler/frontend/maxwell/translate/impl/impl.h
index c994fe803..54c31deb4 100644
--- a/src/shader_recompiler/frontend/maxwell/translate/impl/impl.h
+++ b/src/shader_recompiler/frontend/maxwell/translate/impl/impl.h
@@ -341,10 +341,12 @@ public:
     void XMAD_imm(u64 insn);
 
     [[nodiscard]] IR::U32 X(IR::Reg reg);
+    [[nodiscard]] IR::U64 L(IR::Reg reg);
     [[nodiscard]] IR::F32 F(IR::Reg reg);
     [[nodiscard]] IR::F64 D(IR::Reg reg);
 
     void X(IR::Reg dest_reg, const IR::U32& value);
+    void L(IR::Reg dest_reg, const IR::U64& value);
     void F(IR::Reg dest_reg, const IR::F32& value);
     void D(IR::Reg dest_reg, const IR::F64& value);