12 files changed, 117 insertions, 25 deletions

diff --git a/src/shader_recompiler/backend/glasm/emit_glasm.cpp b/src/shader_recompiler/backend/glasm/emit_glasm.cpp
index fa48ba25c..775dd9e7e 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm.cpp
+++ b/src/shader_recompiler/backend/glasm/emit_glasm.cpp
@@ -2,6 +2,7 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <ranges>
 #include <string>
 #include <tuple>
 
@@ -9,6 +10,7 @@
 #include "shader_recompiler/backend/glasm/emit_context.h"
 #include "shader_recompiler/backend/glasm/emit_glasm.h"
 #include "shader_recompiler/backend/glasm/emit_glasm_instructions.h"
+#include "shader_recompiler/frontend/ir/ir_emitter.h"
 #include "shader_recompiler/frontend/ir/program.h"
 #include "shader_recompiler/profile.h"
 
@@ -175,6 +177,34 @@ void EmitInst(EmitContext& ctx, IR::Inst* inst) {
     throw LogicError("Invalid opcode {}", inst->GetOpcode());
 }
 
+void Precolor(EmitContext& ctx, const IR::Program& program) {
+    for (IR::Block* const block : program.blocks) {
+        for (IR::Inst& phi : block->Instructions() | std::views::take_while(IR::IsPhi)) {
+            switch (phi.Arg(0).Type()) {
+            case IR::Type::U1:
+            case IR::Type::U32:
+            case IR::Type::F32:
+                ctx.reg_alloc.Define(phi);
+                break;
+            case IR::Type::U64:
+            case IR::Type::F64:
+                ctx.reg_alloc.LongDefine(phi);
+                break;
+            default:
+                throw NotImplementedException("Phi node type {}", phi.Type());
+            }
+            const size_t num_args{phi.NumArgs()};
+            for (size_t i = 0; i < num_args; ++i) {
+                IR::IREmitter{*phi.PhiBlock(i)}.PhiMove(phi, phi.Arg(i));
+            }
+            // Add reference to the phi node on the phi predecessor to avoid overwritting it
+            for (size_t i = 0; i < num_args; ++i) {
+                IR::IREmitter{*phi.PhiBlock(i)}.DummyReference(IR::Value{&phi});
+            }
+        }
+    }
+}
+
 void EmitCode(EmitContext& ctx, const IR::Program& program) {
     const auto eval{
         [&](const IR::U1& cond) { return ScalarS32{ctx.reg_alloc.Consume(IR::Value{cond})}; }};
@@ -186,7 +216,9 @@ void EmitCode(EmitContext& ctx, const IR::Program& program) {
             }
             break;
         case IR::AbstractSyntaxNode::Type::If:
-            ctx.Add("MOV.S.CC RC,{};IF NE.x;", eval(node.if_node.cond));
+            ctx.Add("MOV.S.CC RC,{};"
+                    "IF NE.x;",
+                    eval(node.if_node.cond));
             break;
         case IR::AbstractSyntaxNode::Type::EndIf:
             ctx.Add("ENDIF;");
@@ -195,10 +227,30 @@ void EmitCode(EmitContext& ctx, const IR::Program& program) {
             ctx.Add("REP;");
             break;
         case IR::AbstractSyntaxNode::Type::Repeat:
-            ctx.Add("MOV.S.CC RC,{};BRK NE.x;ENDREP;", eval(node.repeat.cond));
+            if (node.repeat.cond.IsImmediate()) {
+                if (node.repeat.cond.U1()) {
+                    ctx.Add("ENDREP;");
+                } else {
+                    ctx.Add("BRK;"
+                            "ENDREP;");
+                }
+            } else {
+                ctx.Add("MOV.S.CC RC,{};"
+                        "BRK (EQ.x);"
+                        "ENDREP;",
+                        eval(node.repeat.cond));
+            }
             break;
         case IR::AbstractSyntaxNode::Type::Break:
-            ctx.Add("MOV.S.CC RC,{};BRK NE.x;", eval(node.repeat.cond));
+            if (node.break_node.cond.IsImmediate()) {
+                if (node.break_node.cond.U1()) {
+                    ctx.Add("BRK;");
+                }
+            } else {
+                ctx.Add("MOV.S.CC RC,{};"
+                        "BRK (NE.x);",
+                        eval(node.break_node.cond));
+            }
             break;
         case IR::AbstractSyntaxNode::Type::Return:
         case IR::AbstractSyntaxNode::Type::Unreachable:
@@ -233,6 +285,7 @@ void SetupOptions(std::string& header, Info info) {
 
 std::string EmitGLASM(const Profile&, IR::Program& program, Bindings&) {
     EmitContext ctx{program};
+    Precolor(ctx, program);
     EmitCode(ctx, program);
     std::string header = "!!NVcp5.0\n"
                          "OPTION NV_internal;";
diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
index 0f7f16e6e..a74e422d6 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
+++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
@@ -22,7 +22,8 @@ class EmitContext;
 void EmitPhi(EmitContext& ctx, IR::Inst& inst);
 void EmitVoid(EmitContext& ctx);
 void EmitIdentity(EmitContext& ctx, IR::Inst& inst, const IR::Value& value);
-void EmitBranchConditionRef(EmitContext&);
+void EmitDummyReference(EmitContext&);
+void EmitPhiMove(EmitContext& ctx, const IR::Value& phi, const IR::Value& value);
 void EmitJoin(EmitContext& ctx);
 void EmitDemoteToHelperInvocation(EmitContext& ctx);
 void EmitBarrier(EmitContext& ctx);
diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp
index f37ad5587..969b91a81 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp
+++ b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp
@@ -17,13 +17,32 @@ namespace Shader::Backend::GLASM {
 
 #define NotImplemented() throw NotImplementedException("GLASM instruction {}", __LINE__)
 
-void EmitPhi(EmitContext& ctx, IR::Inst& inst) {
-    NotImplemented();
-}
+void EmitPhi(EmitContext&, IR::Inst&) {}
 
 void EmitVoid(EmitContext&) {}
 
-void EmitBranchConditionRef(EmitContext&) {}
+void EmitDummyReference(EmitContext&) {}
+
+void EmitPhiMove(EmitContext& ctx, const IR::Value& phi, const IR::Value& value) {
+    if (phi == value) {
+        return;
+    }
+    const Register phi_reg{ctx.reg_alloc.Consume(phi)};
+    const Value eval_value{ctx.reg_alloc.Consume(value)};
+    switch (phi.InstRecursive()->Arg(0).Type()) {
+    case IR::Type::U1:
+    case IR::Type::U32:
+    case IR::Type::F32:
+        ctx.Add("MOV.S {}.x,{};", phi_reg, ScalarS32{eval_value});
+        break;
+    case IR::Type::U64:
+    case IR::Type::F64:
+        ctx.Add("MOV.U64 {}.x,{};", phi_reg, ScalarRegister{eval_value});
+        break;
+    default:
+        throw NotImplementedException("Phi node type {}", phi.Type());
+    }
+}
 
 void EmitJoin(EmitContext& ctx) {
     NotImplemented();
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp
index c22edfec2..7bf8c78de 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp
+++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp
@@ -468,7 +468,11 @@ Id EmitIdentity(EmitContext& ctx, const IR::Value& value) {
     return id;
 }
 
-void EmitBranchConditionRef(EmitContext&) {}
+void EmitDummyReference(EmitContext&) {}
+
+void EmitPhiMove(EmitContext&) {
+    throw LogicError("Unreachable instruction");
+}
 
 void EmitGetZeroFromOp(EmitContext&) {
     throw LogicError("Unreachable instruction");
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
index 2f4f6e59e..0a2b31772 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
@@ -23,7 +23,8 @@ class EmitContext;
 Id EmitPhi(EmitContext& ctx, IR::Inst* inst);
 void EmitVoid(EmitContext& ctx);
 Id EmitIdentity(EmitContext& ctx, const IR::Value& value);
-void EmitBranchConditionRef(EmitContext&);
+void EmitDummyReference(EmitContext&);
+void EmitPhiMove(EmitContext&);
 void EmitJoin(EmitContext& ctx);
 void EmitDemoteToHelperInvocation(EmitContext& ctx);
 void EmitBarrier(EmitContext& ctx);
diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
index eb45aa477..def29143e 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
@@ -61,6 +61,14 @@ F64 IREmitter::Imm64(f64 value) const {
     return F64{Value{value}};
 }
 
+void IREmitter::DummyReference(const Value& value) {
+    Inst(Opcode::DummyReference, value);
+}
+
+void IREmitter::PhiMove(IR::Inst& phi, const Value& value) {
+    Inst(Opcode::PhiMove, Value{&phi}, value);
+}
+
 void IREmitter::Prologue() {
     Inst(Opcode::Prologue);
 }
@@ -69,10 +77,6 @@ void IREmitter::Epilogue() {
     Inst(Opcode::Epilogue);
 }
 
-void IREmitter::BranchConditionRef(const U1& cond) {
-    Inst(Opcode::BranchConditionRef, cond);
-}
-
 void IREmitter::DemoteToHelperInvocation() {
     Inst(Opcode::DemoteToHelperInvocation);
 }
@@ -106,6 +110,9 @@ void IREmitter::SetReg(IR::Reg reg, const U32& value) {
 }
 
 U1 IREmitter::GetPred(IR::Pred pred, bool is_negated) {
+    if (pred == Pred::PT) {
+        return Imm1(!is_negated);
+    }
     const U1 value{Inst<U1>(Opcode::GetPred, pred)};
     if (is_negated) {
         return Inst<U1>(Opcode::LogicalNot, value);
@@ -264,6 +271,9 @@ static U1 GetFlowTest(IREmitter& ir, FlowTest flow_test) {
 U1 IREmitter::Condition(IR::Condition cond) {
     const FlowTest flow_test{cond.GetFlowTest()};
     const auto [pred, is_negated]{cond.GetPred()};
+    if (flow_test == FlowTest::T) {
+        return GetPred(pred, is_negated);
+    }
     return LogicalAnd(GetPred(pred, is_negated), GetFlowTest(*this, flow_test));
 }
 
diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h
index 7a83c33d3..4f7c820fe 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.h
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.h
@@ -32,9 +32,11 @@ public:
     [[nodiscard]] U64 Imm64(s64 value) const;
     [[nodiscard]] F64 Imm64(f64 value) const;
 
+    void DummyReference(const Value& value);
+    void PhiMove(IR::Inst& phi, const Value& value);
+
     void Prologue();
     void Epilogue();
-    void BranchConditionRef(const U1& cond);
     void DemoteToHelperInvocation();
     void EmitVertex(const U32& stream);
     void EndPrimitive(const U32& stream);
diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp
index 364574240..267aebc61 100644
--- a/src/shader_recompiler/frontend/ir/microinstruction.cpp
+++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp
@@ -56,9 +56,10 @@ Inst::~Inst() {
 
 bool Inst::MayHaveSideEffects() const noexcept {
     switch (op) {
+    case Opcode::DummyReference:
+    case Opcode::PhiMove:
     case Opcode::Prologue:
     case Opcode::Epilogue:
-    case Opcode::BranchConditionRef:
     case Opcode::Join:
     case Opcode::DemoteToHelperInvocation:
     case Opcode::Barrier:
diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc
index 75ddb6b6f..6196b867d 100644
--- a/src/shader_recompiler/frontend/ir/opcodes.inc
+++ b/src/shader_recompiler/frontend/ir/opcodes.inc
@@ -6,11 +6,12 @@
 OPCODE(Phi,                                                 Opaque,                                                                                         )
 OPCODE(Identity,                                            Opaque,         Opaque,                                                                         )
 OPCODE(Void,                                                Void,                                                                                           )
+OPCODE(DummyReference,                                      Void,           Opaque,                                                                         )
+OPCODE(PhiMove,                                             Void,           Opaque,         Opaque,                                                         )
 
 // Special operations
 OPCODE(Prologue,                                            Void,                                                                                           )
 OPCODE(Epilogue,                                            Void,                                                                                           )
-OPCODE(BranchConditionRef,                                  Void,           U1,                                                                             )
 OPCODE(Join,                                                Void,                                                                                           )
 OPCODE(DemoteToHelperInvocation,                            Void,                                                                                           )
 OPCODE(EmitVertex,                                          Void,           U32,                                                                            )
diff --git a/src/shader_recompiler/frontend/ir/value.h b/src/shader_recompiler/frontend/ir/value.h
index 2ce49f953..0c6bf684d 100644
--- a/src/shader_recompiler/frontend/ir/value.h
+++ b/src/shader_recompiler/frontend/ir/value.h
@@ -391,4 +391,8 @@ inline f64 Value::F64() const {
     return imm_f64;
 }
 
+[[nodiscard]] inline bool IsPhi(const Inst& inst) {
+    return inst.GetOpcode() == Opcode::Phi;
+}
+
 } // namespace Shader::IR
diff --git a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp
index e7e2e9c82..836d4b8aa 100644
--- a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp
+++ b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp
@@ -704,7 +704,7 @@ private:
                 // Implement if header block
                 IR::IREmitter ir{*current_block};
                 const IR::U1 cond{VisitExpr(ir, *stmt.cond)};
-                ir.BranchConditionRef(cond);
+                ir.DummyReference(cond);
 
                 const size_t if_node_index{syntax_list.size()};
                 syntax_list.emplace_back();
@@ -755,7 +755,7 @@ private:
                 // The continue block is located at the end of the loop
                 IR::IREmitter ir{*continue_block};
                 const IR::U1 cond{VisitExpr(ir, *stmt.cond)};
-                ir.BranchConditionRef(cond);
+                ir.DummyReference(cond);
 
                 IR::Block* const body_block{syntax_list.at(body_block_index).block};
                 loop_header_block->AddBranch(body_block);
@@ -792,7 +792,7 @@ private:
 
                 IR::IREmitter ir{*current_block};
                 const IR::U1 cond{VisitExpr(ir, *stmt.cond)};
-                ir.BranchConditionRef(cond);
+                ir.DummyReference(cond);
                 current_block->AddBranch(break_block);
                 current_block->AddBranch(skip_block);
                 current_block = skip_block;
diff --git a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
index 26eb3a3ab..e54499ba5 100644
--- a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
+++ b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
@@ -138,10 +138,6 @@ IR::Opcode UndefOpcode(IndirectBranchVariable) noexcept {
     return IR::Opcode::UndefU32;
 }
 
-[[nodiscard]] bool IsPhi(const IR::Inst& inst) noexcept {
-    return inst.GetOpcode() == IR::Opcode::Phi;
-}
-
 enum class Status {
     Start,
     SetValue,
@@ -283,7 +279,7 @@ private:
         list.erase(IR::Block::InstructionList::s_iterator_to(phi));
 
         // Find the first non-phi instruction and use it as an insertion point
-        IR::Block::iterator reinsert_point{std::ranges::find_if_not(list, IsPhi)};
+        IR::Block::iterator reinsert_point{std::ranges::find_if_not(list, IR::IsPhi)};
         if (same.IsEmpty()) {
             // The phi is unreachable or in the start block
             // Insert an undefined instruction and make it the phi node replacement