Engines: Implement Accelerate DMA Texture.

3 files changed, 252 insertions, 7 deletions

diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 719edbcfb..f085d53a1 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -172,7 +172,7 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
       buffer_cache(*this, cpu_memory_, buffer_cache_runtime),
       pipeline_cache(*this, device, scheduler, descriptor_pool, update_descriptor_queue,
                      render_pass_cache, buffer_cache, texture_cache, gpu.ShaderNotify()),
-      query_cache{*this, device, scheduler}, accelerate_dma{buffer_cache},
+      query_cache{*this, device, scheduler}, accelerate_dma(buffer_cache, texture_cache, scheduler),
       fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler),
       wfi_event(device.GetLogical().CreateEvent()) {
     scheduler.SetQueryCache(query_cache);
@@ -756,7 +756,9 @@ void RasterizerVulkan::FlushWork() {
     draw_counter = 0;
 }
 
-AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_) : buffer_cache{buffer_cache_} {}
+AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_, TextureCache& texture_cache_,
+                             Scheduler& scheduler_)
+    : buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, scheduler{scheduler_} {}
 
 bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) {
     std::scoped_lock lock{buffer_cache.mutex};
@@ -768,6 +770,234 @@ bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64
     return buffer_cache.DMACopy(src_address, dest_address, amount);
 }
 
+bool AccelerateDMA::ImageToBuffer(const Tegra::DMA::ImageCopy& copy_info,
+                                  const Tegra::DMA::ImageOperand& src,
+                                  const Tegra::DMA::BufferOperand& dst) {
+    std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+    auto query_image = texture_cache.ObtainImage(src, false);
+    if (!query_image) {
+        return false;
+    }
+    auto* image = query_image->first;
+    auto [level, base] = query_image->second;
+    const u32 buffer_size = static_cast<u32>(dst.pitch * dst.height);
+    const auto [buffer, offset] = buffer_cache.ObtainBuffer(
+        dst.address, buffer_size, VideoCommon::ObtainBufferSynchronize::FullSynchronize,
+        VideoCommon::ObtainBufferOperation::MarkAsWritten);
+
+    const bool is_rescaled = image->IsRescaled();
+    if (is_rescaled) {
+        image->ScaleDown();
+    }
+    VkImageSubresourceLayers subresources{
+        .aspectMask = image->AspectMask(),
+        .mipLevel = level,
+        .baseArrayLayer = base,
+        .layerCount = 1,
+    };
+    const u32 bpp = VideoCore::Surface::BytesPerBlock(image->info.format);
+    const auto convert = [old_bpp = src.bytes_per_pixel, bpp](u32 value) {
+        return (old_bpp * value) / bpp;
+    };
+    const u32 base_x = convert(src.params.origin.x.Value());
+    const u32 base_y = src.params.origin.y.Value();
+    const u32 length_x = convert(copy_info.length_x);
+    const u32 length_y = copy_info.length_y;
+    VkOffset3D image_offset{
+        .x = static_cast<s32>(base_x),
+        .y = static_cast<s32>(base_y),
+        .z = 0,
+    };
+    VkExtent3D image_extent{
+        .width = length_x,
+        .height = length_y,
+        .depth = 1,
+    };
+    auto buff_info(dst);
+    buff_info.pitch = convert(dst.pitch);
+    scheduler.RequestOutsideRenderPassOperationContext();
+    scheduler.Record([src_image = image->Handle(), dst_buffer = buffer->Handle(),
+                      buffer_offset = offset, subresources, image_offset, image_extent,
+                      buff_info](vk::CommandBuffer cmdbuf) {
+        const std::array buffer_copy_info{
+            VkBufferImageCopy{
+                .bufferOffset = buffer_offset,
+                .bufferRowLength = buff_info.pitch,
+                .bufferImageHeight = buff_info.height,
+                .imageSubresource = subresources,
+                .imageOffset = image_offset,
+                .imageExtent = image_extent,
+            },
+        };
+        const VkImageSubresourceRange range{
+            .aspectMask = subresources.aspectMask,
+            .baseMipLevel = subresources.mipLevel,
+            .levelCount = 1,
+            .baseArrayLayer = subresources.baseArrayLayer,
+            .layerCount = 1,
+        };
+        static constexpr VkMemoryBarrier WRITE_BARRIER{
+            .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+            .pNext = nullptr,
+            .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
+            .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT,
+        };
+        const std::array pre_barriers{
+            VkImageMemoryBarrier{
+                .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+                .pNext = nullptr,
+                .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+                                 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
+                                 VK_ACCESS_TRANSFER_WRITE_BIT,
+                .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
+                .oldLayout = VK_IMAGE_LAYOUT_GENERAL,
+                .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+                .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .image = src_image,
+                .subresourceRange = range,
+            },
+        };
+        const std::array post_barriers{
+            VkImageMemoryBarrier{
+                .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+                .pNext = nullptr,
+                .srcAccessMask = 0,
+                .dstAccessMask = 0,
+                .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+                .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+                .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .image = src_image,
+                .subresourceRange = range,
+            },
+        };
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+                               0, {}, {}, pre_barriers);
+        cmdbuf.CopyImageToBuffer(src_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_buffer,
+                                 buffer_copy_info);
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                               0, WRITE_BARRIER, nullptr, post_barriers);
+    });
+    if (is_rescaled) {
+        image->ScaleUp(true);
+    }
+    return true;
+}
+
+bool AccelerateDMA::BufferToImage(const Tegra::DMA::ImageCopy& copy_info,
+                                  const Tegra::DMA::BufferOperand& src,
+                                  const Tegra::DMA::ImageOperand& dst) {
+    std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+    auto query_image = texture_cache.ObtainImage(dst, true);
+    if (!query_image) {
+        return false;
+    }
+    auto* image = query_image->first;
+    auto [level, base] = query_image->second;
+    const u32 buffer_size = static_cast<u32>(src.pitch * src.height);
+    const auto [buffer, offset] = buffer_cache.ObtainBuffer(
+        src.address, buffer_size, VideoCommon::ObtainBufferSynchronize::FullSynchronize,
+        VideoCommon::ObtainBufferOperation::DoNothing);
+    const bool is_rescaled = image->IsRescaled();
+    if (is_rescaled) {
+        image->ScaleDown(true);
+    }
+    VkImageSubresourceLayers subresources{
+        .aspectMask = image->AspectMask(),
+        .mipLevel = level,
+        .baseArrayLayer = base,
+        .layerCount = 1,
+    };
+    const u32 bpp = VideoCore::Surface::BytesPerBlock(image->info.format);
+    const auto convert = [old_bpp = dst.bytes_per_pixel, bpp](u32 value) {
+        return (old_bpp * value) / bpp;
+    };
+    const u32 base_x = convert(dst.params.origin.x.Value());
+    const u32 base_y = dst.params.origin.y.Value();
+    const u32 length_x = convert(copy_info.length_x);
+    const u32 length_y = copy_info.length_y;
+    VkOffset3D image_offset{
+        .x = static_cast<s32>(base_x),
+        .y = static_cast<s32>(base_y),
+        .z = 0,
+    };
+    VkExtent3D image_extent{
+        .width = length_x,
+        .height = length_y,
+        .depth = 1,
+    };
+    auto buff_info(src);
+    buff_info.pitch = convert(src.pitch);
+    scheduler.RequestOutsideRenderPassOperationContext();
+    scheduler.Record([dst_image = image->Handle(), src_buffer = buffer->Handle(),
+                      buffer_offset = offset, subresources, image_offset, image_extent,
+                      buff_info](vk::CommandBuffer cmdbuf) {
+        const std::array buffer_copy_info{
+            VkBufferImageCopy{
+                .bufferOffset = buffer_offset,
+                .bufferRowLength = buff_info.pitch,
+                .bufferImageHeight = buff_info.height,
+                .imageSubresource = subresources,
+                .imageOffset = image_offset,
+                .imageExtent = image_extent,
+            },
+        };
+        const VkImageSubresourceRange range{
+            .aspectMask = subresources.aspectMask,
+            .baseMipLevel = subresources.mipLevel,
+            .levelCount = 1,
+            .baseArrayLayer = subresources.baseArrayLayer,
+            .layerCount = 1,
+        };
+        static constexpr VkMemoryBarrier READ_BARRIER{
+            .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+            .pNext = nullptr,
+            .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
+            .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+        };
+        const std::array pre_barriers{
+            VkImageMemoryBarrier{
+                .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+                .pNext = nullptr,
+                .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+                                 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
+                                 VK_ACCESS_TRANSFER_WRITE_BIT,
+                .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
+                .oldLayout = VK_IMAGE_LAYOUT_GENERAL,
+                .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+                .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .image = dst_image,
+                .subresourceRange = range,
+            },
+        };
+        const std::array post_barriers{
+            VkImageMemoryBarrier{
+                .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+                .pNext = nullptr,
+                .srcAccessMask = 0,
+                .dstAccessMask = 0,
+                .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+                .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+                .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+                .image = dst_image,
+                .subresourceRange = range,
+            },
+        };
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+                               0, READ_BARRIER, {}, pre_barriers);
+        cmdbuf.CopyBufferToImage(src_buffer, dst_image, VK_IMAGE_LAYOUT_GENERAL, buffer_copy_info);
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                               0, nullptr, nullptr, post_barriers);
+    });
+    if (is_rescaled) {
+        image->ScaleUp();
+    }
+    return true;
+}
+
 void RasterizerVulkan::UpdateDynamicStates() {
     auto& regs = maxwell3d->regs;
     UpdateViewportsState(regs);
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index a0508b57c..7746c5434 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -45,14 +45,23 @@ class StateTracker;
 
 class AccelerateDMA : public Tegra::Engines::AccelerateDMAInterface {
 public:
-    explicit AccelerateDMA(BufferCache& buffer_cache);
+    explicit AccelerateDMA(BufferCache& buffer_cache, TextureCache& texture_cache,
+                           Scheduler& scheduler);
 
     bool BufferCopy(GPUVAddr start_address, GPUVAddr end_address, u64 amount) override;
 
     bool BufferClear(GPUVAddr src_address, u64 amount, u32 value) override;
 
+    bool ImageToBuffer(const Tegra::DMA::ImageCopy& copy_info, const Tegra::DMA::ImageOperand& src,
+                       const Tegra::DMA::BufferOperand& dst) override;
+
+    bool BufferToImage(const Tegra::DMA::ImageCopy& copy_info, const Tegra::DMA::BufferOperand& src,
+                       const Tegra::DMA::ImageOperand& dst) override;
+
 private:
     BufferCache& buffer_cache;
+    TextureCache& texture_cache;
+    Scheduler& scheduler;
 };
 
 class RasterizerVulkan final : public VideoCore::RasterizerAccelerated,
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 80adb70eb..8a204f93f 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -864,13 +864,19 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src,
     const VkImageAspectFlags src_aspect_mask = src.AspectMask();
     const VkImageAspectFlags dst_aspect_mask = dst.AspectMask();
 
-    std::ranges::transform(copies, vk_in_copies.begin(), [src_aspect_mask](const auto& copy) {
-        return MakeBufferImageCopy(copy, true, src_aspect_mask);
-    });
+    const auto bpp_in = BytesPerBlock(src.info.format) / DefaultBlockWidth(src.info.format);
+    const auto bpp_out = BytesPerBlock(dst.info.format) / DefaultBlockWidth(dst.info.format);
+    std::ranges::transform(copies, vk_in_copies.begin(),
+                           [src_aspect_mask, bpp_in, bpp_out](const auto& copy) {
+                               auto copy2 = copy;
+                               copy2.src_offset.x = (bpp_out * copy.src_offset.x) / bpp_in;
+                               copy2.extent.width = (bpp_out * copy.extent.width) / bpp_in;
+                               return MakeBufferImageCopy(copy2, true, src_aspect_mask);
+                           });
     std::ranges::transform(copies, vk_out_copies.begin(), [dst_aspect_mask](const auto& copy) {
         return MakeBufferImageCopy(copy, false, dst_aspect_mask);
     });
-    const u32 img_bpp = BytesPerBlock(src.info.format);
+    const u32 img_bpp = BytesPerBlock(dst.info.format);
     size_t total_size = 0;
     for (const auto& copy : copies) {
         total_size += copy.extent.width * copy.extent.height * copy.extent.depth * img_bpp;