DMAEngine: Accelerate BufferClear

11 files changed, 115 insertions, 6 deletions

diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index 2871682f6..5f5a59bba 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -164,11 +164,16 @@ public:
     /// Pop asynchronous downloads
     void PopAsyncFlushes();
 
-    [[nodiscard]] bool DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount);
+    bool DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount);
+
+    bool DMAClear(GPUVAddr src_address, u64 amount, u32 value);
 
     /// Return true when a CPU region is modified from the GPU
     [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
 
+    /// Return true when a region is registered on the cache
+    [[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
+
     /// Return true when a CPU region is modified from the CPU
     [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
 
@@ -469,8 +474,8 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
     if (!cpu_src_address || !cpu_dest_address) {
         return false;
     }
-    const bool source_dirty = IsRegionGpuModified(*cpu_src_address, amount);
-    const bool dest_dirty = IsRegionGpuModified(*cpu_dest_address, amount);
+    const bool source_dirty = IsRegionRegistered(*cpu_src_address, amount);
+    const bool dest_dirty = IsRegionRegistered(*cpu_dest_address, amount);
     if (!source_dirty && !dest_dirty) {
         return false;
     }
@@ -515,7 +520,7 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
     }
 
     runtime.CopyBuffer(dest_buffer, src_buffer, copies);
-    if (source_dirty) {
+    if (IsRegionGpuModified(*cpu_src_address, amount)) {
         dest_buffer.MarkRegionAsGpuModified(*cpu_dest_address, amount);
     }
     std::vector<u8> tmp_buffer(amount);
@@ -525,6 +530,37 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
 }
 
 template <class P>
+bool BufferCache<P>::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) {
+    const std::optional<VAddr> cpu_dst_address = gpu_memory.GpuToCpuAddress(dst_address);
+    if (!cpu_dst_address) {
+        return false;
+    }
+    const bool dest_dirty = IsRegionRegistered(*cpu_dst_address, amount);
+    if (!dest_dirty) {
+        return false;
+    }
+
+    const IntervalType subtract_interval{*cpu_dst_address, *cpu_dst_address + amount * sizeof(u32)};
+    uncommitted_ranges.subtract(subtract_interval);
+    for (auto& interval_set : committed_ranges) {
+        interval_set.subtract(subtract_interval);
+    }
+    common_ranges.subtract(subtract_interval);
+
+    const size_t size = amount * sizeof(u32);
+    BufferId buffer;
+    do {
+        has_deleted_buffers = false;
+        buffer = FindBuffer(*cpu_dst_address, static_cast<u32>(size));
+    } while (has_deleted_buffers);
+
+    auto& dest_buffer = slot_buffers[buffer];
+    const u32 offset = static_cast<u32>(*cpu_dst_address - dest_buffer.CpuAddr());
+    runtime.ClearBuffer(dest_buffer, offset, size, value);
+    return true;
+}
+
+template <class P>
 void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr,
                                                u32 size) {
     const std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
@@ -782,6 +818,27 @@ bool BufferCache<P>::IsRegionGpuModified(VAddr addr, size_t size) {
 }
 
 template <class P>
+bool BufferCache<P>::IsRegionRegistered(VAddr addr, size_t size) {
+    const VAddr end_addr = addr + size;
+    const u64 page_end = Common::DivCeil(end_addr, PAGE_SIZE);
+    for (u64 page = addr >> PAGE_BITS; page < page_end;) {
+        const BufferId buffer_id = page_table[page];
+        if (!buffer_id) {
+            ++page;
+            continue;
+        }
+        Buffer& buffer = slot_buffers[buffer_id];
+        const VAddr buf_start_addr = buffer.CpuAddr();
+        const VAddr buf_end_addr = buf_start_addr + buffer.SizeBytes();
+        if (buf_start_addr < end_addr && addr < buf_end_addr) {
+            return true;
+        }
+        page = Common::DivCeil(end_addr, PAGE_SIZE);
+    }
+    return false;
+}
+
+template <class P>
 bool BufferCache<P>::IsRegionCpuModified(VAddr addr, size_t size) {
     const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE);
     for (u64 page = addr >> PAGE_BITS; page < page_end;) {
diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp
index 24481952b..81becb88a 100644
--- a/src/video_core/engines/maxwell_dma.cpp
+++ b/src/video_core/engines/maxwell_dma.cpp
@@ -87,9 +87,11 @@ void MaxwellDMA::CopyPitchToPitch() {
         // TODO: allow multisized components.
         if (is_buffer_clear) {
             ASSERT(regs.remap_const.component_size_minus_one == 3);
+            accelerate.BufferClear(regs.offset_out, regs.line_length_in, regs.remap_consta_value);
             std::vector<u32> tmp_buffer(regs.line_length_in, regs.remap_consta_value);
-            memory_manager.WriteBlock(regs.offset_out, reinterpret_cast<u8*>(tmp_buffer.data()),
-                                      regs.line_length_in * sizeof(u32));
+            memory_manager.WriteBlockUnsafe(regs.offset_out,
+                                            reinterpret_cast<u8*>(tmp_buffer.data()),
+                                            regs.line_length_in * sizeof(u32));
             return;
         }
         UNIMPLEMENTED_IF(regs.launch_dma.remap_enable != 0);
diff --git a/src/video_core/engines/maxwell_dma.h b/src/video_core/engines/maxwell_dma.h
index 4ed0d0996..d3329b0f8 100644
--- a/src/video_core/engines/maxwell_dma.h
+++ b/src/video_core/engines/maxwell_dma.h
@@ -31,6 +31,8 @@ class AccelerateDMAInterface {
 public:
     /// Write the value to the register identified by method.
     virtual bool BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) = 0;
+
+    virtual bool BufferClear(GPUVAddr src_address, u64 amount, u32 value) = 0;
 };
 
 /**
diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp
index c225d1fc9..c4189fb60 100644
--- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp
@@ -98,6 +98,12 @@ void BufferCacheRuntime::CopyBuffer(Buffer& dst_buffer, Buffer& src_buffer,
     }
 }
 
+void BufferCacheRuntime::ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value) {
+    glClearNamedBufferSubData(dest_buffer.Handle(), GL_R32UI, static_cast<GLintptr>(offset),
+                              static_cast<GLsizeiptr>(size / sizeof(u32)), GL_RGBA, GL_UNSIGNED_INT,
+                              &value);
+}
+
 void BufferCacheRuntime::BindIndexBuffer(Buffer& buffer, u32 offset, u32 size) {
     if (has_unified_vertex_buffers) {
         buffer.MakeResident(GL_READ_ONLY);
diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h
index d8b20a9af..fe91aa452 100644
--- a/src/video_core/renderer_opengl/gl_buffer_cache.h
+++ b/src/video_core/renderer_opengl/gl_buffer_cache.h
@@ -57,6 +57,8 @@ public:
     void CopyBuffer(Buffer& dst_buffer, Buffer& src_buffer,
                     std::span<const VideoCommon::BufferCopy> copies);
 
+    void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value);
+
     void BindIndexBuffer(Buffer& buffer, u32 offset, u32 size);
 
     void BindVertexBuffer(u32 index, Buffer& buffer, u32 offset, u32 size, u32 stride);
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 82c84127a..ceb3abcb2 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -1407,4 +1407,9 @@ bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64
     return buffer_cache.DMACopy(src_address, dest_address, amount);
 }
 
+bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) {
+    std::scoped_lock lock{buffer_cache.mutex};
+    return buffer_cache.DMAClear(src_address, amount, value);
+}
+
 } // namespace OpenGL
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h
index ccee9ba33..d30ad698f 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer.h
@@ -65,6 +65,8 @@ public:
 
     bool BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) override;
 
+    bool BufferClear(GPUVAddr src_address, u64 amount, u32 value) override;
+
 private:
     BufferCache& buffer_cache;
 };
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
index 0df4e1a1c..11e86174e 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
@@ -136,6 +136,30 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer,
     });
 }
 
+void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t size, u32 value) {
+    static constexpr VkMemoryBarrier READ_BARRIER{
+        .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+        .pNext = nullptr,
+        .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
+        .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
+    };
+    static constexpr VkMemoryBarrier WRITE_BARRIER{
+        .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+        .pNext = nullptr,
+        .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
+        .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT,
+    };
+
+    scheduler.RequestOutsideRenderPassOperationContext();
+    scheduler.Record([dest_buffer, offset, size, value](vk::CommandBuffer cmdbuf) {
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
+                               0, READ_BARRIER);
+        cmdbuf.FillBuffer(dest_buffer, offset, size, value);
+        cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                               0, WRITE_BARRIER);
+    });
+}
+
 void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format,
                                          u32 base_vertex, u32 num_indices, VkBuffer buffer,
                                          u32 offset, [[maybe_unused]] u32 size) {
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index 982e92191..60d49386e 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -60,6 +60,8 @@ public:
     void CopyBuffer(VkBuffer src_buffer, VkBuffer dst_buffer,
                     std::span<const VideoCommon::BufferCopy> copies);
 
+    void ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t size, u32 value);
+
     void BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, u32 num_indices,
                          u32 base_vertex, VkBuffer buffer, u32 offset, u32 size);
 
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index e378a5679..fd9eed3e3 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -704,6 +704,11 @@ void RasterizerVulkan::FlushWork() {
 
 AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_) : buffer_cache{buffer_cache_} {}
 
+bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) {
+    std::scoped_lock lock{buffer_cache.mutex};
+    return buffer_cache.DMAClear(src_address, amount, value);
+}
+
 bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) {
     std::scoped_lock lock{buffer_cache.mutex};
     return buffer_cache.DMACopy(src_address, dest_address, amount);
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index 3a78de258..2065209be 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -56,6 +56,8 @@ public:
 
     bool BufferCopy(GPUVAddr start_address, GPUVAddr end_address, u64 amount) override;
 
+    bool BufferClear(GPUVAddr src_address, u64 amount, u32 value) override;
+
 private:
     BufferCache& buffer_cache;
 };