8 files changed, 151 insertions, 41 deletions

diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_image.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_image.cpp
index 6e940bd5a..ad39f44c3 100644
--- a/src/shader_recompiler/backend/glsl/emit_glsl_image.cpp
+++ b/src/shader_recompiler/backend/glsl/emit_glsl_image.cpp
@@ -449,7 +449,7 @@ void EmitImageGatherDref(EmitContext& ctx, IR::Inst& inst, const IR::Value& inde
 }
 
 void EmitImageFetch(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
-                    std::string_view coords, std::string_view offset, std::string_view lod,
+                    std::string_view coords, const IR::Value& offset, std::string_view lod,
                     std::string_view ms) {
     const auto info{inst.Flags<IR::TextureInstInfo>()};
     if (info.has_bias) {
@@ -470,9 +470,9 @@ void EmitImageFetch(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
         const auto int_coords{CoordsCastToInt(coords, info)};
         if (!ms.empty()) {
             ctx.Add("{}=texelFetch({},{},int({}));", texel, texture, int_coords, ms);
-        } else if (!offset.empty()) {
+        } else if (!offset.IsEmpty()) {
             ctx.Add("{}=texelFetchOffset({},{},int({}),{});", texel, texture, int_coords, lod,
-                    CoordsCastToInt(offset, info));
+                    GetOffsetVec(ctx, offset));
         } else {
             if (info.type == TextureType::Buffer) {
                 ctx.Add("{}=texelFetch({},int({}));", texel, texture, coords);
@@ -485,10 +485,10 @@ void EmitImageFetch(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
     if (!ms.empty()) {
         throw NotImplementedException("EmitImageFetch Sparse MSAA samples");
     }
-    if (!offset.empty()) {
+    if (!offset.IsEmpty()) {
         ctx.AddU1("{}=sparseTexelsResidentARB(sparseTexelFetchOffsetARB({},{},int({}),{},{}));",
-                  *sparse_inst, texture, CastToIntVec(coords, info), lod,
-                  CastToIntVec(offset, info), texel);
+                  *sparse_inst, texture, CastToIntVec(coords, info), lod, GetOffsetVec(ctx, offset),
+                  texel);
     } else {
         ctx.AddU1("{}=sparseTexelsResidentARB(sparseTexelFetchARB({},{},int({}),{}));",
                   *sparse_inst, texture, CastToIntVec(coords, info), lod, texel);
diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h
index 8d0a65047..acebaa785 100644
--- a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h
+++ b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h
@@ -651,7 +651,7 @@ void EmitImageGatherDref(EmitContext& ctx, IR::Inst& inst, const IR::Value& inde
                          std::string_view coords, const IR::Value& offset, const IR::Value& offset2,
                          std::string_view dref);
 void EmitImageFetch(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
-                    std::string_view coords, std::string_view offset, std::string_view lod,
+                    std::string_view coords, const IR::Value& offset, std::string_view lod,
                     std::string_view ms);
 void EmitImageQueryDimensions(EmitContext& ctx, IR::Inst& inst, const IR::Value& index,
                               std::string_view lod, const IR::Value& skip_mips);
diff --git a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp
index b2ceeefc4..c5ac7b8f2 100644
--- a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp
+++ b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp
@@ -608,8 +608,8 @@ std::string EmitContext::DefineGlobalMemoryFunctions() {
         const auto aligned_low_addr{fmt::format("{}&{}", addr_xy[0], ssbo_align_mask)};
         const auto aligned_addr{fmt::format("uvec2({},{})", aligned_low_addr, addr_xy[1])};
         const auto addr_pack{fmt::format("packUint2x32({})", aligned_addr)};
-        const auto addr_statment{fmt::format("uint64_t {}={};", ssbo_addr, addr_pack)};
-        func += addr_statment;
+        const auto addr_statement{fmt::format("uint64_t {}={};", ssbo_addr, addr_pack)};
+        func += addr_statement;
 
         const auto size_vec{fmt::format("uvec2({},{})", size_xy[0], size_xy[1])};
         const auto comp_lhs{fmt::format("(addr>={})", ssbo_addr)};
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
index 800754554..64a4e0e55 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_image.cpp
@@ -12,6 +12,11 @@ namespace Shader::Backend::SPIRV {
 namespace {
 class ImageOperands {
 public:
+    [[maybe_unused]] static constexpr bool ImageSampleOffsetAllowed = false;
+    [[maybe_unused]] static constexpr bool ImageGatherOffsetAllowed = true;
+    [[maybe_unused]] static constexpr bool ImageFetchOffsetAllowed = false;
+    [[maybe_unused]] static constexpr bool ImageGradientOffsetAllowed = false;
+
     explicit ImageOperands(EmitContext& ctx, bool has_bias, bool has_lod, bool has_lod_clamp,
                            Id lod, const IR::Value& offset) {
         if (has_bias) {
@@ -22,7 +27,7 @@ public:
             const Id lod_value{has_lod_clamp ? ctx.OpCompositeExtract(ctx.F32[1], lod, 0) : lod};
             Add(spv::ImageOperandsMask::Lod, lod_value);
         }
-        AddOffset(ctx, offset);
+        AddOffset(ctx, offset, ImageSampleOffsetAllowed);
         if (has_lod_clamp) {
             const Id lod_clamp{has_bias ? ctx.OpCompositeExtract(ctx.F32[1], lod, 1) : lod};
             Add(spv::ImageOperandsMask::MinLod, lod_clamp);
@@ -55,20 +60,17 @@ public:
         Add(spv::ImageOperandsMask::ConstOffsets, offsets);
     }
 
-    explicit ImageOperands(Id offset, Id lod, Id ms) {
+    explicit ImageOperands(Id lod, Id ms) {
         if (Sirit::ValidId(lod)) {
             Add(spv::ImageOperandsMask::Lod, lod);
         }
-        if (Sirit::ValidId(offset)) {
-            Add(spv::ImageOperandsMask::Offset, offset);
-        }
         if (Sirit::ValidId(ms)) {
             Add(spv::ImageOperandsMask::Sample, ms);
         }
     }
 
     explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives,
-                           u32 num_derivatives, Id offset, Id lod_clamp) {
+                           u32 num_derivatives, const IR::Value& offset, Id lod_clamp) {
         if (!Sirit::ValidId(derivatives)) {
             throw LogicError("Derivatives must be present");
         }
@@ -83,16 +85,14 @@ public:
         const Id derivatives_Y{ctx.OpCompositeConstruct(
             ctx.F32[num_derivatives], std::span{deriv_y_accum.data(), deriv_y_accum.size()})};
         Add(spv::ImageOperandsMask::Grad, derivatives_X, derivatives_Y);
-        if (Sirit::ValidId(offset)) {
-            Add(spv::ImageOperandsMask::Offset, offset);
-        }
+        AddOffset(ctx, offset, ImageGradientOffsetAllowed);
         if (has_lod_clamp) {
             Add(spv::ImageOperandsMask::MinLod, lod_clamp);
         }
     }
 
     explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives_1, Id derivatives_2,
-                           Id offset, Id lod_clamp) {
+                           const IR::Value& offset, Id lod_clamp) {
         if (!Sirit::ValidId(derivatives_1) || !Sirit::ValidId(derivatives_2)) {
             throw LogicError("Derivatives must be present");
         }
@@ -111,9 +111,7 @@ public:
         const Id derivatives_id2{ctx.OpCompositeConstruct(
             ctx.F32[3], std::span{deriv_2_accum.data(), deriv_2_accum.size()})};
         Add(spv::ImageOperandsMask::Grad, derivatives_id1, derivatives_id2);
-        if (Sirit::ValidId(offset)) {
-            Add(spv::ImageOperandsMask::Offset, offset);
-        }
+        AddOffset(ctx, offset, ImageGradientOffsetAllowed);
         if (has_lod_clamp) {
             Add(spv::ImageOperandsMask::MinLod, lod_clamp);
         }
@@ -132,7 +130,7 @@ public:
     }
 
 private:
-    void AddOffset(EmitContext& ctx, const IR::Value& offset) {
+    void AddOffset(EmitContext& ctx, const IR::Value& offset, bool runtime_offset_allowed) {
         if (offset.IsEmpty()) {
             return;
         }
@@ -165,7 +163,9 @@ private:
                 break;
             }
         }
-        Add(spv::ImageOperandsMask::Offset, ctx.Def(offset));
+        if (runtime_offset_allowed) {
+            Add(spv::ImageOperandsMask::Offset, ctx.Def(offset));
+        }
     }
 
     void Add(spv::ImageOperandsMask new_mask, Id value) {
@@ -311,6 +311,37 @@ Id ImageGatherSubpixelOffset(EmitContext& ctx, const IR::TextureInstInfo& info,
         return coords;
     }
 }
+
+void AddOffsetToCoordinates(EmitContext& ctx, const IR::TextureInstInfo& info, Id& coords,
+                            Id offset) {
+    if (!Sirit::ValidId(offset)) {
+        return;
+    }
+
+    Id result_type{};
+    switch (info.type) {
+    case TextureType::Buffer:
+    case TextureType::Color1D:
+    case TextureType::ColorArray1D: {
+        result_type = ctx.U32[1];
+        break;
+    }
+    case TextureType::Color2D:
+    case TextureType::Color2DRect:
+    case TextureType::ColorArray2D: {
+        result_type = ctx.U32[2];
+        break;
+    }
+    case TextureType::Color3D: {
+        result_type = ctx.U32[3];
+        break;
+    }
+    case TextureType::ColorCube:
+    case TextureType::ColorArrayCube:
+        return;
+    }
+    coords = ctx.OpIAdd(result_type, coords, offset);
+}
 } // Anonymous namespace
 
 Id EmitBindlessImageSampleImplicitLod(EmitContext&) {
@@ -496,6 +527,7 @@ Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
 Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id offset,
                   Id lod, Id ms) {
     const auto info{inst->Flags<IR::TextureInstInfo>()};
+    AddOffsetToCoordinates(ctx, info, coords, offset);
     if (info.type == TextureType::Buffer) {
         lod = Id{};
     }
@@ -503,7 +535,7 @@ Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id c
         // This image is multisampled, lod must be implicit
         lod = Id{};
     }
-    const ImageOperands operands(offset, lod, ms);
+    const ImageOperands operands(lod, ms);
     return Emit(&EmitContext::OpImageSparseFetch, &EmitContext::OpImageFetch, ctx, inst, ctx.F32[4],
                 TextureImage(ctx, info, index), coords, operands.MaskOptional(), operands.Span());
 }
@@ -548,13 +580,13 @@ Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, I
 }
 
 Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
-                     Id derivatives, Id offset, Id lod_clamp) {
+                     Id derivatives, const IR::Value& offset, Id lod_clamp) {
     const auto info{inst->Flags<IR::TextureInstInfo>()};
-    const auto operands =
-        info.num_derivatives == 3
-            ? ImageOperands(ctx, info.has_lod_clamp != 0, derivatives, offset, {}, lod_clamp)
-            : ImageOperands(ctx, info.has_lod_clamp != 0, derivatives, info.num_derivatives, offset,
-                            lod_clamp);
+    const auto operands = info.num_derivatives == 3
+                              ? ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
+                                              ctx.Def(offset), {}, lod_clamp)
+                              : ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
+                                              info.num_derivatives, offset, lod_clamp);
     return Emit(&EmitContext::OpImageSparseSampleExplicitLod,
                 &EmitContext::OpImageSampleExplicitLod, ctx, inst, ctx.F32[4],
                 Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
index 7d34575c8..5c01b1012 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h
@@ -543,7 +543,7 @@ Id EmitImageQueryDimensions(EmitContext& ctx, IR::Inst* inst, const IR::Value& i
                             const IR::Value& skip_mips);
 Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
 Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
-                     Id derivatives, Id offset, Id lod_clamp);
+                     Id derivatives, const IR::Value& offset, Id lod_clamp);
 Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
 void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color);
 Id EmitIsTextureScaled(EmitContext& ctx, const IR::Value& index);
diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp
index 8693801c7..bdcbccfde 100644
--- a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp
+++ b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp
@@ -65,6 +65,14 @@ void WriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value&
     WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32, sizeof(u32),
                  &StorageDefinitions::U32, index_offset);
 }
+
+void WriteStorageByCasLoop(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
+                           Id value, Id bit_offset, Id bit_count) {
+    const Id pointer{StoragePointer(ctx, binding, offset, ctx.storage_types.U32, sizeof(u32),
+                                    &StorageDefinitions::U32)};
+    ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_storage_cas_loop_func, pointer, value, bit_offset,
+                       bit_count);
+}
 } // Anonymous namespace
 
 void EmitLoadGlobalU8(EmitContext&) {
@@ -219,26 +227,42 @@ Id EmitLoadStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Valu
 
 void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
                         Id value) {
-    WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U8, value), ctx.storage_types.U8,
-                 sizeof(u8), &StorageDefinitions::U8);
+    if (ctx.profile.support_int8) {
+        WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U8, value), ctx.storage_types.U8,
+                     sizeof(u8), &StorageDefinitions::U8);
+    } else {
+        WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
+    }
 }
 
 void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
                         Id value) {
-    WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S8, value), ctx.storage_types.S8,
-                 sizeof(s8), &StorageDefinitions::S8);
+    if (ctx.profile.support_int8) {
+        WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S8, value), ctx.storage_types.S8,
+                     sizeof(s8), &StorageDefinitions::S8);
+    } else {
+        WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
+    }
 }
 
 void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
                          Id value) {
-    WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U16, value), ctx.storage_types.U16,
-                 sizeof(u16), &StorageDefinitions::U16);
+    if (ctx.profile.support_int16) {
+        WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U16, value), ctx.storage_types.U16,
+                     sizeof(u16), &StorageDefinitions::U16);
+    } else {
+        WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
+    }
 }
 
 void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
                          Id value) {
-    WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S16, value), ctx.storage_types.S16,
-                 sizeof(s16), &StorageDefinitions::S16);
+    if (ctx.profile.support_int16) {
+        WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S16, value), ctx.storage_types.S16,
+                     sizeof(s16), &StorageDefinitions::S16);
+    } else {
+        WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
+    }
 }
 
 void EmitWriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
index 89ebab08e..a27f2f73a 100644
--- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
+++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
@@ -480,6 +480,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf
     DefineTextures(program.info, texture_binding, bindings.texture_scaling_index);
     DefineImages(program.info, image_binding, bindings.image_scaling_index);
     DefineAttributeMemAccess(program.info);
+    DefineWriteStorageCasLoopFunction(program.info);
     DefineGlobalMemoryFunctions(program.info);
     DefineRescalingInput(program.info);
     DefineRenderArea(program.info);
@@ -877,6 +878,56 @@ void EmitContext::DefineAttributeMemAccess(const Info& info) {
     }
 }
 
+void EmitContext::DefineWriteStorageCasLoopFunction(const Info& info) {
+    if (profile.support_int8 && profile.support_int16) {
+        return;
+    }
+    if (!info.uses_int8 && !info.uses_int16) {
+        return;
+    }
+
+    AddCapability(spv::Capability::VariablePointersStorageBuffer);
+
+    const Id ptr_type{TypePointer(spv::StorageClass::StorageBuffer, U32[1])};
+    const Id func_type{TypeFunction(void_id, ptr_type, U32[1], U32[1], U32[1])};
+    const Id func{OpFunction(void_id, spv::FunctionControlMask::MaskNone, func_type)};
+    const Id pointer{OpFunctionParameter(ptr_type)};
+    const Id value{OpFunctionParameter(U32[1])};
+    const Id bit_offset{OpFunctionParameter(U32[1])};
+    const Id bit_count{OpFunctionParameter(U32[1])};
+
+    AddLabel();
+    const Id scope_device{Const(1u)};
+    const Id ordering_relaxed{u32_zero_value};
+    const Id body_label{OpLabel()};
+    const Id continue_label{OpLabel()};
+    const Id endloop_label{OpLabel()};
+    const Id beginloop_label{OpLabel()};
+    OpBranch(beginloop_label);
+
+    AddLabel(beginloop_label);
+    OpLoopMerge(endloop_label, continue_label, spv::LoopControlMask::MaskNone);
+    OpBranch(body_label);
+
+    AddLabel(body_label);
+    const Id expected_value{OpLoad(U32[1], pointer)};
+    const Id desired_value{OpBitFieldInsert(U32[1], expected_value, value, bit_offset, bit_count)};
+    const Id actual_value{OpAtomicCompareExchange(U32[1], pointer, scope_device, ordering_relaxed,
+                                                  ordering_relaxed, desired_value, expected_value)};
+    const Id store_successful{OpIEqual(U1, expected_value, actual_value)};
+    OpBranchConditional(store_successful, endloop_label, continue_label);
+
+    AddLabel(endloop_label);
+    OpReturn();
+
+    AddLabel(continue_label);
+    OpBranch(beginloop_label);
+
+    OpFunctionEnd();
+
+    write_storage_cas_loop_func = func;
+}
+
 void EmitContext::DefineGlobalMemoryFunctions(const Info& info) {
     if (!info.uses_global_memory || !profile.support_int64) {
         return;
@@ -1440,7 +1491,7 @@ void EmitContext::DefineInputs(const IR::Program& program) {
         if (profile.support_vertex_instance_id) {
             instance_id = DefineInput(*this, U32[1], true, spv::BuiltIn::InstanceId);
             if (loads[IR::Attribute::BaseInstance]) {
-                base_instance = DefineInput(*this, U32[1], true, spv::BuiltIn::BaseVertex);
+                base_instance = DefineInput(*this, U32[1], true, spv::BuiltIn::BaseInstance);
             }
         } else {
             instance_index = DefineInput(*this, U32[1], true, spv::BuiltIn::InstanceIndex);
diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h
index 56019ad89..40adcb6b6 100644
--- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h
+++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h
@@ -325,6 +325,8 @@ public:
     Id f32x2_min_cas{};
     Id f32x2_max_cas{};
 
+    Id write_storage_cas_loop_func{};
+
     Id load_global_func_u32{};
     Id load_global_func_u32x2{};
     Id load_global_func_u32x4{};
@@ -372,6 +374,7 @@ private:
     void DefineTextures(const Info& info, u32& binding, u32& scaling_index);
     void DefineImages(const Info& info, u32& binding, u32& scaling_index);
     void DefineAttributeMemAccess(const Info& info);
+    void DefineWriteStorageCasLoopFunction(const Info& info);
     void DefineGlobalMemoryFunctions(const Info& info);
     void DefineRescalingInput(const Info& info);
     void DefineRescalingInputPushConstant();