6 files changed, 258 insertions, 47 deletions

diff --git a/src/shader_recompiler/backend/glasm/emit_glasm.cpp b/src/shader_recompiler/backend/glasm/emit_glasm.cpp
index 0c591f73c..d7a08e4b3 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm.cpp
+++ b/src/shader_recompiler/backend/glasm/emit_glasm.cpp
@@ -283,6 +283,9 @@ void SetupOptions(std::string& header, Info info) {
     if (info.uses_subgroup_shuffles) {
         header += "OPTION NV_shader_thread_shuffle;";
     }
+    if (info.uses_sparse_residency) {
+        header += "OPTION EXT_sparse_texture2;";
+    }
     const auto non_zero_frag_colors{info.stores_frag_color | std::views::drop(1)};
     if (std::ranges::find(non_zero_frag_colors, true) != non_zero_frag_colors.end()) {
         header += "OPTION ARB_draw_buffers;";
diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_image.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_image.cpp
index 2af5483d9..3d76ab315 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm_image.cpp
+++ b/src/shader_recompiler/backend/glasm/emit_glasm_image.cpp
@@ -9,6 +9,34 @@
 
 namespace Shader::Backend::GLASM {
 namespace {
+struct ScopedRegister {
+    ScopedRegister() = default;
+    ScopedRegister(RegAlloc& reg_alloc_) : reg_alloc{&reg_alloc_}, reg{reg_alloc->AllocReg()} {}
+
+    ~ScopedRegister() {
+        if (reg_alloc) {
+            reg_alloc->FreeReg(reg);
+        }
+    }
+
+    ScopedRegister& operator=(ScopedRegister&& rhs) noexcept {
+        if (reg_alloc) {
+            reg_alloc->FreeReg(reg);
+        }
+        reg_alloc = std::exchange(rhs.reg_alloc, nullptr);
+        reg = rhs.reg;
+    }
+
+    ScopedRegister(ScopedRegister&& rhs) noexcept
+        : reg_alloc{std::exchange(rhs.reg_alloc, nullptr)}, reg{rhs.reg} {}
+
+    ScopedRegister& operator=(const ScopedRegister&) = delete;
+    ScopedRegister(const ScopedRegister&) = delete;
+
+    RegAlloc* reg_alloc{};
+    Register reg;
+};
+
 std::string Texture([[maybe_unused]] EmitContext& ctx, IR::TextureInstInfo info,
                     [[maybe_unused]] const IR::Value& index) {
     // FIXME
@@ -36,6 +64,61 @@ std::string_view TextureType(IR::TextureInstInfo info) {
     }
     throw InvalidArgument("Invalid texture type {}", info.type.Value());
 }
+
+std::string_view ShadowTextureType(IR::TextureInstInfo info) {
+    switch (info.type) {
+    case TextureType::Color1D:
+        return "SHADOW1D";
+    case TextureType::ColorArray1D:
+        return "SHADOWARRAY1D";
+    case TextureType::Color2D:
+        return "SHADOW2D";
+    case TextureType::ColorArray2D:
+        return "SHADOWARRAY2D";
+    case TextureType::Color3D:
+        return "SHADOW3D";
+    case TextureType::ColorCube:
+        return "SHADOWCUBE";
+    case TextureType::ColorArrayCube:
+        return "SHADOWARRAYCUBE";
+    case TextureType::Buffer:
+        return "SHADOWBUFFER";
+    }
+    throw InvalidArgument("Invalid texture type {}", info.type.Value());
+}
+
+std::string Offset(EmitContext& ctx, const IR::Value& offset) {
+    if (offset.IsEmpty()) {
+        return "";
+    }
+    return fmt::format(",offset({})", Register{ctx.reg_alloc.Consume(offset)});
+}
+
+std::pair<std::string, ScopedRegister> Coord(EmitContext& ctx, const IR::Value& coord) {
+    if (coord.IsImmediate()) {
+        ScopedRegister scoped_reg(ctx.reg_alloc);
+        return {fmt::to_string(scoped_reg.reg), std::move(scoped_reg)};
+    }
+    std::string coord_vec{fmt::to_string(Register{ctx.reg_alloc.Consume(coord)})};
+    if (coord.InstRecursive()->HasUses()) {
+        // Move non-dead coords to a separate register, although this should never happen because
+        // vectors are only assembled for immediate texture instructions
+        ctx.Add("MOV.F RC,{};", coord_vec);
+        coord_vec = "RC";
+    }
+    return {std::move(coord_vec), ScopedRegister{}};
+}
+
+void StoreSparse(EmitContext& ctx, IR::Inst* sparse_inst) {
+    if (!sparse_inst) {
+        return;
+    }
+    const Register sparse_ret{ctx.reg_alloc.Define(*sparse_inst)};
+    ctx.Add("MOV.S {},-1;"
+            "MOV.S {}(NONRESIDENT),0;",
+            sparse_ret, sparse_ret);
+    sparse_inst->Invalidate();
+}
 } // Anonymous namespace
 
 void EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
@@ -46,17 +129,8 @@ void EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Valu
     const std::string_view lod_clamp_mod{info.has_lod_clamp ? ".LODCLAMP" : ""};
     const std::string_view type{TextureType(info)};
     const std::string texture{Texture(ctx, info, index)};
-    std::string offset_vec;
-    if (!offset.IsEmpty()) {
-        offset_vec = fmt::format(",offset({})", Register{ctx.reg_alloc.Consume(offset)});
-    }
-    std::string coord_vec{fmt::to_string(Register{ctx.reg_alloc.Consume(coord)})};
-    if (coord.InstRecursive()->HasUses()) {
-        // Move non-dead coords to a separate register, although this should never happen because
-        // vectors are only assembled for immediate texture instructions
-        ctx.Add("MOV.F RC,{};", coord_vec);
-        coord_vec = "RC";
-    }
+    const std::string offset_vec{Offset(ctx, offset)};
+    const auto [coord_vec, coord_alloc]{Coord(ctx, coord)};
     const Register ret{ctx.reg_alloc.Define(inst)};
     if (info.has_bias) {
         if (info.type == TextureType::ColorArrayCube) {
@@ -83,38 +157,172 @@ void EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Valu
                     type, offset_vec);
         }
     }
-    if (sparse_inst) {
-        const Register sparse_ret{ctx.reg_alloc.Define(*sparse_inst)};
-        ctx.Add("MOV.S {},-1;"
-                "MOV.S {}(NONRESIDENT),0;",
-                sparse_ret, sparse_ret);
-        sparse_inst->Invalidate();
-    }
+    StoreSparse(ctx, sparse_inst);
 }
 
-void EmitImageSampleExplicitLod([[maybe_unused]] EmitContext& ctx, [[maybe_unused]] IR::Inst& inst,
-                                [[maybe_unused]] const IR::Value& index,
-                                [[maybe_unused]] Register coord, [[maybe_unused]] Register lod_lc,
-                                [[maybe_unused]] const IR::Value& offset) {
-    throw NotImplementedException("GLASM instruction");
+void EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
+                                const IR::Value& coord, ScalarF32 lod, const IR::Value& offset) {
+    const auto info{inst.Flags<IR::TextureInstInfo>()};
+    const auto sparse_inst{inst.GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)};
+    const std::string_view sparse_mod{sparse_inst ? ".SPARSE" : ""};
+    const std::string_view type{TextureType(info)};
+    const std::string texture{Texture(ctx, info, index)};
+    const std::string offset_vec{Offset(ctx, offset)};
+    const auto [coord_vec, coord_alloc]{Coord(ctx, coord)};
+    const Register ret{ctx.reg_alloc.Define(inst)};
+    if (info.type == TextureType::ColorArrayCube) {
+        ctx.Add("TXL.F{} {},{},{},{},ARRAYCUBE{};", sparse_mod, ret, coord_vec, lod, texture,
+                offset_vec);
+    } else {
+        ctx.Add("MOV.F {}.w,{};"
+                "TXL.F{} {},{},{},{}{};",
+                coord_vec, lod, sparse_mod, ret, coord_vec, texture, type, offset_vec);
+    }
+    StoreSparse(ctx, sparse_inst);
 }
 
-void EmitImageSampleDrefImplicitLod([[maybe_unused]] EmitContext& ctx,
-                                    [[maybe_unused]] IR::Inst& inst,
-                                    [[maybe_unused]] const IR::Value& index,
-                                    [[maybe_unused]] Register coord, [[maybe_unused]] Register dref,
-                                    [[maybe_unused]] Register bias_lc,
-                                    [[maybe_unused]] const IR::Value& offset) {
-    throw NotImplementedException("GLASM instruction");
+void EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
+                                    const IR::Value& coord, ScalarF32 dref, Register bias_lc,
+                                    const IR::Value& offset) {
+    const auto info{inst.Flags<IR::TextureInstInfo>()};
+    const auto sparse_inst{inst.GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)};
+    const std::string_view sparse_mod{sparse_inst ? ".SPARSE" : ""};
+    const std::string_view type{ShadowTextureType(info)};
+    const std::string texture{Texture(ctx, info, index)};
+    const std::string offset_vec{Offset(ctx, offset)};
+    const auto [coord_vec, coord_alloc]{Coord(ctx, coord)};
+    const Register ret{ctx.reg_alloc.Define(inst)};
+    if (info.has_bias) {
+        if (info.has_lod_clamp) {
+            switch (info.type) {
+            case TextureType::Color1D:
+            case TextureType::ColorArray1D:
+            case TextureType::Color2D:
+                ctx.Add("MOV.F {}.z,{};"
+                        "MOV.F {}.w,{}.x;"
+                        "TXB.F.LODCLAMP{} {},{},{}.y,{},{}{};",
+                        coord_vec, dref, coord_vec, bias_lc, sparse_mod, ret, coord_vec, bias_lc,
+                        texture, type, offset_vec);
+                break;
+            case TextureType::ColorArray2D:
+            case TextureType::ColorCube:
+                ctx.Add("MOV.F {}.w,{};"
+                        "TXB.F.LODCLAMP{} {},{},{},{},{}{};",
+                        coord_vec, dref, sparse_mod, ret, coord_vec, bias_lc, texture, type,
+                        offset_vec);
+                break;
+            default:
+                throw NotImplementedException("Invalid type {} with bias and lod clamp",
+                                              info.type.Value());
+            }
+        } else {
+            switch (info.type) {
+            case TextureType::Color1D:
+            case TextureType::ColorArray1D:
+            case TextureType::Color2D:
+                ctx.Add("MOV.F {}.z,{};"
+                        "MOV.F {}.w,{}.x;"
+                        "TXB.F{} {},{},{},{}{};",
+                        coord_vec, dref, coord_vec, bias_lc, sparse_mod, ret, coord_vec, texture,
+                        type, offset_vec);
+                break;
+            case TextureType::ColorArray2D:
+            case TextureType::ColorCube:
+                ctx.Add("MOV.F {}.w,{};"
+                        "TXB.F{} {},{},{},{},{}{};",
+                        coord_vec, dref, sparse_mod, ret, coord_vec, bias_lc, texture, type,
+                        offset_vec);
+                break;
+            case TextureType::ColorArrayCube: {
+                const ScopedRegister pair{ctx.reg_alloc};
+                ctx.Add("MOV.F {}.x,{};"
+                        "MOV.F {}.y,{}.x;"
+                        "TXB.F{} {},{},{},{},{}{};",
+                        pair.reg, dref, pair.reg, bias_lc, sparse_mod, ret, coord_vec, pair.reg,
+                        texture, type, offset_vec);
+                break;
+            }
+            default:
+                throw NotImplementedException("Invalid type {}", info.type.Value());
+            }
+        }
+    } else {
+        if (info.has_lod_clamp) {
+            if (info.type != TextureType::ColorArrayCube) {
+                const bool w_swizzle{info.type == TextureType::ColorArray2D ||
+                                     info.type == TextureType::ColorCube};
+                const char dref_swizzle{w_swizzle ? 'w' : 'z'};
+                ctx.Add("MOV.F {}.{},{};"
+                        "TEX.F.LODCLAMP{} {},{},{},{},{}{};",
+                        coord_vec, dref_swizzle, dref, sparse_mod, ret, coord_vec, bias_lc, texture,
+                        type, offset_vec);
+            } else {
+                const ScopedRegister pair{ctx.reg_alloc};
+                ctx.Add("MOV.F {}.x,{};"
+                        "MOV.F {}.y,{};"
+                        "TEX.F.LODCLAMP{} {},{},{},{},{}{};",
+                        pair.reg, dref, pair.reg, bias_lc, sparse_mod, ret, coord_vec, pair.reg,
+                        texture, type, offset_vec);
+            }
+        } else {
+            if (info.type != TextureType::ColorArrayCube) {
+                const bool w_swizzle{info.type == TextureType::ColorArray2D ||
+                                     info.type == TextureType::ColorCube};
+                const char dref_swizzle{w_swizzle ? 'w' : 'z'};
+                ctx.Add("MOV.F {}.{},{};"
+                        "TEX.F{} {},{},{},{}{};",
+                        coord_vec, dref_swizzle, dref, sparse_mod, ret, coord_vec, texture, type,
+                        offset_vec);
+            } else {
+                const ScopedRegister pair{ctx.reg_alloc};
+                ctx.Add("TEX.F{} {},{},{},{},{}{};", sparse_mod, ret, coord_vec, dref, texture,
+                        type, offset_vec);
+            }
+        }
+    }
+    StoreSparse(ctx, sparse_inst);
 }
 
-void EmitImageSampleDrefExplicitLod([[maybe_unused]] EmitContext& ctx,
-                                    [[maybe_unused]] IR::Inst& inst,
-                                    [[maybe_unused]] const IR::Value& index,
-                                    [[maybe_unused]] Register coord, [[maybe_unused]] Register dref,
-                                    [[maybe_unused]] Register lod_lc,
-                                    [[maybe_unused]] const IR::Value& offset) {
-    throw NotImplementedException("GLASM instruction");
+void EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
+                                    const IR::Value& coord, ScalarF32 dref, ScalarF32 lod,
+                                    const IR::Value& offset) {
+    const auto info{inst.Flags<IR::TextureInstInfo>()};
+    const auto sparse_inst{inst.GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)};
+    const std::string_view sparse_mod{sparse_inst ? ".SPARSE" : ""};
+    const std::string_view type{ShadowTextureType(info)};
+    const std::string texture{Texture(ctx, info, index)};
+    const std::string offset_vec{Offset(ctx, offset)};
+    const auto [coord_vec, coord_alloc]{Coord(ctx, coord)};
+    const Register ret{ctx.reg_alloc.Define(inst)};
+    switch (info.type) {
+    case TextureType::Color1D:
+    case TextureType::ColorArray1D:
+    case TextureType::Color2D:
+        ctx.Add("MOV.F {}.z,{};"
+                "MOV.F {}.w,{};"
+                "TXL.F{} {},{},{},{}{};",
+                coord_vec, dref, coord_vec, lod, sparse_mod, ret, coord_vec, texture, type,
+                offset_vec);
+        break;
+    case TextureType::ColorArray2D:
+    case TextureType::ColorCube:
+        ctx.Add("MOV.F {}.w,{};"
+                "TXL.F{} {},{},{},{},{}{};",
+                coord_vec, dref, sparse_mod, ret, coord_vec, lod, texture, type, offset_vec);
+        break;
+    case TextureType::ColorArrayCube: {
+        const ScopedRegister pair{ctx.reg_alloc};
+        ctx.Add("MOV.F {}.x,{};"
+                "MOV.F {}.y,{};"
+                "TXL.F{} {},{},{},{},{}{};",
+                pair.reg, dref, pair.reg, lod, sparse_mod, ret, coord_vec, pair.reg, texture, type,
+                offset_vec);
+        break;
+    }
+    default:
+        throw NotImplementedException("Invalid type {}", info.type.Value());
+    }
+    StoreSparse(ctx, sparse_inst);
 }
 
 void EmitImageGather([[maybe_unused]] EmitContext& ctx, [[maybe_unused]] IR::Inst& inst,
diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
index a128f9ac4..54e7fab3c 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
+++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h
@@ -527,12 +527,12 @@ void EmitBoundImageWrite(EmitContext&);
 void EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
                                 const IR::Value& coord, Register bias_lc, const IR::Value& offset);
 void EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
-                                Register coord, Register lod_lc, const IR::Value& offset);
+                                const IR::Value& coord, ScalarF32 lod, const IR::Value& offset);
 void EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
-                                    Register coord, Register dref, Register bias_lc,
+                                    const IR::Value& coord, ScalarF32 dref, Register bias_lc,
                                     const IR::Value& offset);
 void EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
-                                    Register coord, Register dref, Register lod_lc,
+                                    const IR::Value& coord, ScalarF32 dref, ScalarF32 lod,
                                     const IR::Value& offset);
 void EmitImageGather(EmitContext& ctx, IR::Inst& inst, const IR::Value& index, Register coord,
                      const IR::Value& offset, const IR::Value& offset2);
diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp
index 0c6a6e1c8..dd307a9a3 100644
--- a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp
+++ b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp
@@ -38,7 +38,7 @@ void Store(EmitContext& ctx, const IR::Value& binding, ScalarU32 offset, ValueTy
 void Load(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding, ScalarU32 offset,
           std::string_view size) {
     const Register ret{ctx.reg_alloc.Define(inst)};
-    StorageOp(ctx, binding, offset, fmt::format("STORE.{} {},DC.x;", size, ret),
+    StorageOp(ctx, binding, offset, fmt::format("LOAD.{} {},DC.x;", size, ret),
               fmt::format("MOV.U {},{{0,0,0,0}};", ret));
 }
 } // Anonymous namespace
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
index 5832104df..99b883746 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
@@ -337,9 +337,9 @@ Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value&
 }
 
 Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
-                              Id lod_lc, const IR::Value& offset) {
+                              Id lod, const IR::Value& offset) {
     const auto info{inst->Flags<IR::TextureInstInfo>()};
-    const ImageOperands operands(ctx, false, true, info.has_lod_clamp != 0, lod_lc, offset);
+    const ImageOperands operands(ctx, false, true, false, lod, offset);
     return Emit(&EmitContext::OpImageSparseSampleExplicitLod,
                 &EmitContext::OpImageSampleExplicitLod, ctx, inst, ctx.F32[4],
                 Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
@@ -356,9 +356,9 @@ Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Va
 }
 
 Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
-                                  Id coords, Id dref, Id lod_lc, const IR::Value& offset) {
+                                  Id coords, Id dref, Id lod, const IR::Value& offset) {
     const auto info{inst->Flags<IR::TextureInstInfo>()};
-    const ImageOperands operands(ctx, false, true, info.has_lod_clamp != 0, lod_lc, offset);
+    const ImageOperands operands(ctx, false, true, false, lod, offset);
     return Emit(&EmitContext::OpImageSparseSampleDrefExplicitLod,
                 &EmitContext::OpImageSampleDrefExplicitLod, ctx, inst, ctx.F32[1],
                 Texture(ctx, info, index), coords, dref, operands.Mask(), operands.Span());
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
index 0a2b31772..22260d2a9 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
@@ -491,11 +491,11 @@ Id EmitBoundImageWrite(EmitContext&);
 Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
                               Id bias_lc, const IR::Value& offset);
 Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
-                              Id lod_lc, const IR::Value& offset);
+                              Id lod, const IR::Value& offset);
 Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
                                   Id coords, Id dref, Id bias_lc, const IR::Value& offset);
 Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
-                                  Id coords, Id dref, Id lod_lc, const IR::Value& offset);
+                                  Id coords, Id dref, Id lod, const IR::Value& offset);
 Id EmitImageGather(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
                    const IR::Value& offset, const IR::Value& offset2);
 Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,