shader: Properly store phi on Inst

6 files changed, 132 insertions, 75 deletions

diff --git a/src/shader_recompiler/frontend/ir/basic_block.cpp b/src/shader_recompiler/frontend/ir/basic_block.cpp
index 1a5d82135..50c6a83cd 100644
--- a/src/shader_recompiler/frontend/ir/basic_block.cpp
+++ b/src/shader_recompiler/frontend/ir/basic_block.cpp
@@ -129,26 +129,21 @@ std::string DumpBlock(const Block& block, const std::map<const Block*, size_t>&
         } else {
             ret += fmt::format("         {}", op); // '%00000 = ' -> 1 + 5 + 3 = 9 spaces
         }
-        if (op == Opcode::Phi) {
-            size_t val_index{0};
-            for (const auto& [phi_block, phi_val] : inst.PhiOperands()) {
-                ret += val_index != 0 ? ", " : " ";
-                ret += fmt::format("[ {}, {} ]", ArgToIndex(block_to_index, inst_to_index, phi_val),
-                                   BlockToIndex(block_to_index, phi_block));
-                ++val_index;
+        const size_t arg_count{NumArgsOf(op)};
+        for (size_t arg_index = 0; arg_index < arg_count; ++arg_index) {
+            const Value arg{inst.Arg(arg_index)};
+            const std::string arg_str{ArgToIndex(block_to_index, inst_to_index, arg)};
+            ret += arg_index != 0 ? ", " : " ";
+            if (op == Opcode::Phi) {
+                ret += fmt::format("[ {}, {} ]", arg_index,
+                                   BlockToIndex(block_to_index, inst.PhiBlock(arg_index)));
+            } else {
+                ret += arg_str;
             }
-        } else {
-            const size_t arg_count{NumArgsOf(op)};
-            for (size_t arg_index = 0; arg_index < arg_count; ++arg_index) {
-                const Value arg{inst.Arg(arg_index)};
-                ret += arg_index != 0 ? ", " : " ";
-                ret += ArgToIndex(block_to_index, inst_to_index, arg);
-
-                const Type actual_type{arg.Type()};
-                const Type expected_type{ArgTypeOf(op, arg_index)};
-                if (!AreTypesCompatible(actual_type, expected_type)) {
-                    ret += fmt::format("<type error: {} != {}>", actual_type, expected_type);
-                }
+            const Type actual_type{arg.Type()};
+            const Type expected_type{ArgTypeOf(op, arg_index)};
+            if (!AreTypesCompatible(actual_type, expected_type)) {
+                ret += fmt::format("<type error: {} != {}>", actual_type, expected_type);
             }
         }
         if (TypeOf(op) != Type::Void) {
diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp
index de953838c..e7ca92039 100644
--- a/src/shader_recompiler/frontend/ir/microinstruction.cpp
+++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp
@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include <algorithm>
+#include <memory>
 
 #include "shader_recompiler/exception.h"
 #include "shader_recompiler/frontend/ir/microinstruction.h"
@@ -30,6 +31,22 @@ static void RemovePseudoInstruction(IR::Inst*& inst, IR::Opcode expected_opcode)
     inst = nullptr;
 }
 
+Inst::Inst(IR::Opcode op_, u64 flags_) noexcept : op{op_}, flags{flags_} {
+    if (op == Opcode::Phi) {
+        std::construct_at(&phi_args);
+    } else {
+        std::construct_at(&args);
+    }
+}
+
+Inst::~Inst() {
+    if (op == Opcode::Phi) {
+        std::destroy_at(&phi_args);
+    } else {
+        std::destroy_at(&args);
+    }
+}
+
 bool Inst::MayHaveSideEffects() const noexcept {
     switch (op) {
     case Opcode::Branch:
@@ -71,7 +88,10 @@ bool Inst::IsPseudoInstruction() const noexcept {
     }
 }
 
-bool Inst::AreAllArgsImmediates() const noexcept {
+bool Inst::AreAllArgsImmediates() const {
+    if (op == Opcode::Phi) {
+        throw LogicError("Testing for all arguments are immediates on phi instruction");
+    }
     return std::all_of(args.begin(), args.begin() + NumArgs(),
                        [](const IR::Value& value) { return value.IsImmediate(); });
 }
@@ -101,7 +121,7 @@ Inst* Inst::GetAssociatedPseudoOperation(IR::Opcode opcode) {
 }
 
 size_t Inst::NumArgs() const {
-    return NumArgsOf(op);
+    return op == Opcode::Phi ? phi_args.size() : NumArgsOf(op);
 }
 
 IR::Type Inst::Type() const {
@@ -109,13 +129,23 @@ IR::Type Inst::Type() const {
 }
 
 Value Inst::Arg(size_t index) const {
-    if (index >= NumArgsOf(op)) {
-        throw InvalidArgument("Out of bounds argument index {} in opcode {}", index, op);
+    if (op == Opcode::Phi) {
+        if (index >= phi_args.size()) {
+            throw InvalidArgument("Out of bounds argument index {} in phi instruction", index);
+        }
+        return phi_args[index].second;
+    } else {
+        if (index >= NumArgsOf(op)) {
+            throw InvalidArgument("Out of bounds argument index {} in opcode {}", index, op);
+        }
+        return args[index];
     }
-    return args[index];
 }
 
 void Inst::SetArg(size_t index, Value value) {
+    if (op == Opcode::Phi) {
+        throw LogicError("Setting argument on a phi instruction");
+    }
     if (index >= NumArgsOf(op)) {
         throw InvalidArgument("Out of bounds argument index {} in opcode {}", index, op);
     }
@@ -128,15 +158,21 @@ void Inst::SetArg(size_t index, Value value) {
     args[index] = value;
 }
 
-std::span<const std::pair<Block*, Value>> Inst::PhiOperands() const noexcept {
-    return phi_operands;
+Block* Inst::PhiBlock(size_t index) const {
+    if (op != Opcode::Phi) {
+        throw LogicError("{} is not a Phi instruction", op);
+    }
+    if (index >= phi_args.size()) {
+        throw InvalidArgument("Out of bounds argument index {} in phi instruction");
+    }
+    return phi_args[index].first;
 }
 
 void Inst::AddPhiOperand(Block* predecessor, const Value& value) {
     if (!value.IsImmediate()) {
         Use(value);
     }
-    phi_operands.emplace_back(predecessor, value);
+    phi_args.emplace_back(predecessor, value);
 }
 
 void Inst::Invalidate() {
@@ -145,18 +181,22 @@ void Inst::Invalidate() {
 }
 
 void Inst::ClearArgs() {
-    for (auto& value : args) {
-        if (!value.IsImmediate()) {
-            UndoUse(value);
+    if (op == Opcode::Phi) {
+        for (auto& pair : phi_args) {
+            IR::Value& value{pair.second};
+            if (!value.IsImmediate()) {
+                UndoUse(value);
+            }
         }
-        value = {};
-    }
-    for (auto& [phi_block, phi_op] : phi_operands) {
-        if (!phi_op.IsImmediate()) {
-            UndoUse(phi_op);
+        phi_args.clear();
+    } else {
+        for (auto& value : args) {
+            if (!value.IsImmediate()) {
+                UndoUse(value);
+            }
+            value = {};
         }
     }
-    phi_operands.clear();
 }
 
 void Inst::ReplaceUsesWith(Value replacement) {
@@ -167,24 +207,29 @@ void Inst::ReplaceUsesWith(Value replacement) {
     if (!replacement.IsImmediate()) {
         Use(replacement);
     }
-    args[0] = replacement;
+    if (op == Opcode::Phi) {
+        phi_args[0].second = replacement;
+    } else {
+        args[0] = replacement;
+    }
 }
 
 void Inst::Use(const Value& value) {
-    ++value.Inst()->use_count;
+    Inst* const inst{value.Inst()};
+    ++inst->use_count;
 
     switch (op) {
     case Opcode::GetZeroFromOp:
-        SetPseudoInstruction(value.Inst()->zero_inst, this);
+        SetPseudoInstruction(inst->zero_inst, this);
         break;
     case Opcode::GetSignFromOp:
-        SetPseudoInstruction(value.Inst()->sign_inst, this);
+        SetPseudoInstruction(inst->sign_inst, this);
         break;
     case Opcode::GetCarryFromOp:
-        SetPseudoInstruction(value.Inst()->carry_inst, this);
+        SetPseudoInstruction(inst->carry_inst, this);
         break;
     case Opcode::GetOverflowFromOp:
-        SetPseudoInstruction(value.Inst()->overflow_inst, this);
+        SetPseudoInstruction(inst->overflow_inst, this);
         break;
     default:
         break;
@@ -192,20 +237,21 @@ void Inst::Use(const Value& value) {
 }
 
 void Inst::UndoUse(const Value& value) {
-    --value.Inst()->use_count;
+    Inst* const inst{value.Inst()};
+    --inst->use_count;
 
     switch (op) {
     case Opcode::GetZeroFromOp:
-        RemovePseudoInstruction(value.Inst()->zero_inst, Opcode::GetZeroFromOp);
+        RemovePseudoInstruction(inst->zero_inst, Opcode::GetZeroFromOp);
         break;
     case Opcode::GetSignFromOp:
-        RemovePseudoInstruction(value.Inst()->sign_inst, Opcode::GetSignFromOp);
+        RemovePseudoInstruction(inst->sign_inst, Opcode::GetSignFromOp);
         break;
     case Opcode::GetCarryFromOp:
-        RemovePseudoInstruction(value.Inst()->carry_inst, Opcode::GetCarryFromOp);
+        RemovePseudoInstruction(inst->carry_inst, Opcode::GetCarryFromOp);
         break;
     case Opcode::GetOverflowFromOp:
-        RemovePseudoInstruction(value.Inst()->overflow_inst, Opcode::GetOverflowFromOp);
+        RemovePseudoInstruction(inst->overflow_inst, Opcode::GetOverflowFromOp);
         break;
     default:
         break;
diff --git a/src/shader_recompiler/frontend/ir/microinstruction.h b/src/shader_recompiler/frontend/ir/microinstruction.h
index 80baffb2e..ddf0f90a9 100644
--- a/src/shader_recompiler/frontend/ir/microinstruction.h
+++ b/src/shader_recompiler/frontend/ir/microinstruction.h
@@ -6,8 +6,8 @@
 
 #include <array>
 #include <cstring>
-#include <span>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 #include <boost/intrusive/list.hpp>
@@ -25,7 +25,14 @@ constexpr size_t MAX_ARG_COUNT = 4;
 
 class Inst : public boost::intrusive::list_base_hook<> {
 public:
-    explicit Inst(Opcode op_, u64 flags_) noexcept : op{op_}, flags{flags_} {}
+    explicit Inst(Opcode op_, u64 flags_) noexcept;
+    ~Inst();
+
+    Inst& operator=(const Inst&) = delete;
+    Inst(const Inst&) = delete;
+
+    Inst& operator=(Inst&&) = delete;
+    Inst(Inst&&) = delete;
 
     /// Get the number of uses this instruction has.
     [[nodiscard]] int UseCount() const noexcept {
@@ -50,26 +57,26 @@ public:
     [[nodiscard]] bool IsPseudoInstruction() const noexcept;
 
     /// Determines if all arguments of this instruction are immediates.
-    [[nodiscard]] bool AreAllArgsImmediates() const noexcept;
+    [[nodiscard]] bool AreAllArgsImmediates() const;
 
     /// Determines if there is a pseudo-operation associated with this instruction.
     [[nodiscard]] bool HasAssociatedPseudoOperation() const noexcept;
     /// Gets a pseudo-operation associated with this instruction
     [[nodiscard]] Inst* GetAssociatedPseudoOperation(IR::Opcode opcode);
 
-    /// Get the number of arguments this instruction has.
-    [[nodiscard]] size_t NumArgs() const;
-
     /// Get the type this instruction returns.
     [[nodiscard]] IR::Type Type() const;
 
+    /// Get the number of arguments this instruction has.
+    [[nodiscard]] size_t NumArgs() const;
+
     /// Get the value of a given argument index.
     [[nodiscard]] Value Arg(size_t index) const;
     /// Set the value of a given argument index.
     void SetArg(size_t index, Value value);
 
-    /// Get an immutable span to the phi operands.
-    [[nodiscard]] std::span<const std::pair<Block*, Value>> PhiOperands() const noexcept;
+    /// Get a pointer to the block of a phi argument.
+    [[nodiscard]] Block* PhiBlock(size_t index) const;
     /// Add phi operand to a phi instruction.
     void AddPhiOperand(Block* predecessor, const Value& value);
 
@@ -87,18 +94,26 @@ public:
     }
 
 private:
+    struct NonTriviallyDummy {
+        NonTriviallyDummy() noexcept {}
+    };
+
     void Use(const Value& value);
     void UndoUse(const Value& value);
 
     IR::Opcode op{};
     int use_count{};
-    std::array<Value, MAX_ARG_COUNT> args{};
+    u64 flags{};
+    union {
+        NonTriviallyDummy dummy{};
+        std::array<Value, MAX_ARG_COUNT> args;
+        std::vector<std::pair<Block*, Value>> phi_args;
+    };
     Inst* zero_inst{};
     Inst* sign_inst{};
     Inst* carry_inst{};
     Inst* overflow_inst{};
-    std::vector<std::pair<Block*, Value>> phi_operands;
-    u64 flags{};
 };
+static_assert(sizeof(Inst) <= 128, "Inst size unintentionally increased its size");
 
 } // namespace Shader::IR
diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc
index 6eb105d92..82b04f37c 100644
--- a/src/shader_recompiler/frontend/ir/opcodes.inc
+++ b/src/shader_recompiler/frontend/ir/opcodes.inc
@@ -3,9 +3,9 @@
 // Refer to the license.txt file included.
 
 //     opcode name,                                         return type,    arg1 type,      arg2 type,      arg3 type,      arg4 type,      ...
+OPCODE(Phi,                                                 Opaque,                                                                         )
 OPCODE(Void,                                                Void,                                                                           )
 OPCODE(Identity,                                            Opaque,         Opaque,                                                         )
-OPCODE(Phi,                                                 Opaque,         /*todo*/                                                        )
 
 // Control flow
 OPCODE(Branch,                                              Void,           Label,                                                          )
diff --git a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
index 7713e3ba9..15a9db90a 100644
--- a/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
+++ b/src/shader_recompiler/ir_opt/ssa_rewrite_pass.cpp
@@ -104,32 +104,34 @@ private:
             val = ReadVariable(variable, preds.front());
         } else {
             // Break potential cycles with operandless phi
-            val = IR::Value{&*block->PrependNewInst(block->begin(), IR::Opcode::Phi)};
+            IR::Inst& phi_inst{*block->PrependNewInst(block->begin(), IR::Opcode::Phi)};
+            val = IR::Value{&phi_inst};
             WriteVariable(variable, block, val);
-            val = AddPhiOperands(variable, val, block);
+            val = AddPhiOperands(variable, phi_inst, block);
         }
         WriteVariable(variable, block, val);
         return val;
     }
 
-    IR::Value AddPhiOperands(auto variable, const IR::Value& phi, IR::Block* block) {
+    IR::Value AddPhiOperands(auto variable, IR::Inst& phi, IR::Block* block) {
         for (IR::Block* const pred : block->ImmediatePredecessors()) {
-            phi.Inst()->AddPhiOperand(pred, ReadVariable(variable, pred));
+            phi.AddPhiOperand(pred, ReadVariable(variable, pred));
         }
         return TryRemoveTrivialPhi(phi, block, UndefOpcode(variable));
     }
 
-    IR::Value TryRemoveTrivialPhi(const IR::Value& phi, IR::Block* block, IR::Opcode undef_opcode) {
+    IR::Value TryRemoveTrivialPhi(IR::Inst& phi, IR::Block* block, IR::Opcode undef_opcode) {
         IR::Value same;
-        for (const auto& pair : phi.Inst()->PhiOperands()) {
-            const IR::Value& op{pair.second};
-            if (op == same || op == phi) {
+        const size_t num_args{phi.NumArgs()};
+        for (size_t arg_index = 0; arg_index < num_args; ++arg_index) {
+            const IR::Value& op{phi.Arg(arg_index)};
+            if (op == same || op == IR::Value{&phi}) {
                 // Unique value or self-reference
                 continue;
             }
             if (!same.IsEmpty()) {
                 // The phi merges at least two values: not trivial
-                return phi;
+                return IR::Value{&phi};
             }
             same = op;
         }
@@ -139,7 +141,7 @@ private:
             same = IR::Value{&*block->PrependNewInst(first_not_phi, undef_opcode)};
         }
         // Reroute all uses of phi to same and remove phi
-        phi.Inst()->ReplaceUsesWith(same);
+        phi.ReplaceUsesWith(same);
         // TODO: Try to recursively remove all phi users, which might have become trivial
         return same;
     }
diff --git a/src/shader_recompiler/object_pool.h b/src/shader_recompiler/object_pool.h
index 7c65bbd92..a573add32 100644
--- a/src/shader_recompiler/object_pool.h
+++ b/src/shader_recompiler/object_pool.h
@@ -6,6 +6,7 @@
 
 #include <memory>
 #include <type_traits>
+#include <utility>
 
 namespace Shader {
 
@@ -31,14 +32,12 @@ public:
 
     void ReleaseContents() {
         Chunk* chunk{&root};
-        if (chunk) {
-            const size_t free_objects{chunk->free_objects};
-            if (free_objects == chunk_size) {
+        while (chunk) {
+            if (chunk->free_objects == chunk_size) {
                 break;
             }
-            chunk->free_objects = chunk_size;
-            for (size_t obj_id = free_objects; obj_id < chunk_size; ++obj_id) {
-                chunk->storage[obj_id].object.~T();
+            for (; chunk->free_objects < chunk_size; ++chunk->free_objects) {
+                chunk->storage[chunk->free_objects].object.~T();
             }
             chunk = chunk->next.get();
         }