DMA & InlineToMemory Engines Rework.

7 files changed, 127 insertions, 53 deletions

diff --git a/src/video_core/engines/engine_upload.cpp b/src/video_core/engines/engine_upload.cpp
index 6ff5b1eca..a34819234 100644
--- a/src/video_core/engines/engine_upload.cpp
+++ b/src/video_core/engines/engine_upload.cpp
@@ -3,6 +3,7 @@
 
 #include <cstring>
 
+#include "common/algorithm.h"
 #include "common/assert.h"
 #include "video_core/engines/engine_upload.h"
 #include "video_core/memory_manager.h"
@@ -34,21 +35,48 @@ void State::ProcessData(const u32 data, const bool is_last_call) {
     if (!is_last_call) {
         return;
     }
+    ProcessData(inner_buffer);
+}
+
+void State::ProcessData(const u32* data, size_t num_data) {
+    std::span<const u8> read_buffer(reinterpret_cast<const u8*>(data), num_data * sizeof(u32));
+    ProcessData(read_buffer);
+}
+
+void State::ProcessData(std::span<const u8> read_buffer) {
     const GPUVAddr address{regs.dest.Address()};
     if (is_linear) {
-        rasterizer->AccelerateInlineToMemory(address, copy_size, inner_buffer);
+        if (regs.line_count == 1) {
+            rasterizer->AccelerateInlineToMemory(address, copy_size, read_buffer);
+        } else {
+            for (u32 line = 0; line < regs.line_count; ++line) {
+                const GPUVAddr dest_line = address + static_cast<size_t>(line) * regs.dest.pitch;
+                memory_manager.WriteBlockUnsafe(
+                    dest_line, read_buffer.data() + static_cast<size_t>(line) * regs.line_length_in,
+                    regs.line_length_in);
+            }
+            memory_manager.InvalidateRegion(address, regs.dest.pitch * regs.line_count);
+        }
     } else {
-        UNIMPLEMENTED_IF(regs.dest.z != 0);
-        UNIMPLEMENTED_IF(regs.dest.depth != 1);
-        UNIMPLEMENTED_IF(regs.dest.BlockWidth() != 0);
-        UNIMPLEMENTED_IF(regs.dest.BlockDepth() != 0);
+        u32 width = regs.dest.width;
+        u32 x_elements = regs.line_length_in;
+        u32 x_offset = regs.dest.x;
+        const u32 bpp_shift = Common::FoldRight(
+            4U, [](u32 x, u32 y) { return std::min(x, static_cast<u32>(std::countr_zero(y))); },
+            width, x_elements, x_offset, static_cast<u32>(address));
+        width >>= bpp_shift;
+        x_elements >>= bpp_shift;
+        x_offset >>= bpp_shift;
+        const u32 bytes_per_pixel = 1U << bpp_shift;
         const std::size_t dst_size = Tegra::Texture::CalculateSize(
-            true, 1, regs.dest.width, regs.dest.height, 1, regs.dest.BlockHeight(), 0);
+            true, bytes_per_pixel, width, regs.dest.height, regs.dest.depth,
+            regs.dest.BlockHeight(), regs.dest.BlockDepth());
         tmp_buffer.resize(dst_size);
         memory_manager.ReadBlock(address, tmp_buffer.data(), dst_size);
-        Tegra::Texture::SwizzleKepler(regs.dest.width, regs.dest.height, regs.dest.x, regs.dest.y,
-                                      regs.dest.BlockHeight(), copy_size, inner_buffer.data(),
-                                      tmp_buffer.data());
+        Tegra::Texture::SwizzleSubrect(tmp_buffer, read_buffer, bytes_per_pixel, width,
+                                       regs.dest.height, regs.dest.depth, x_offset, regs.dest.y,
+                                       x_elements, regs.line_count, regs.dest.BlockHeight(),
+                                       regs.dest.BlockDepth(), regs.line_length_in);
         memory_manager.WriteBlock(address, tmp_buffer.data(), dst_size);
     }
 }
diff --git a/src/video_core/engines/engine_upload.h b/src/video_core/engines/engine_upload.h
index 94ff3314a..f08f6e36a 100644
--- a/src/video_core/engines/engine_upload.h
+++ b/src/video_core/engines/engine_upload.h
@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include <span>
 #include <vector>
 #include "common/bit_field.h"
 #include "common/common_types.h"
@@ -33,7 +34,7 @@ struct Registers {
         u32 width;
         u32 height;
         u32 depth;
-        u32 z;
+        u32 layer;
         u32 x;
         u32 y;
 
@@ -62,11 +63,14 @@ public:
 
     void ProcessExec(bool is_linear_);
     void ProcessData(u32 data, bool is_last_call);
+    void ProcessData(const u32* data, size_t num_data);
 
     /// Binds a rasterizer to this engine.
     void BindRasterizer(VideoCore::RasterizerInterface* rasterizer);
 
 private:
+    void ProcessData(std::span<const u8> read_buffer);
+
     u32 write_offset = 0;
     u32 copy_size = 0;
     std::vector<u8> inner_buffer;
diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp
index 5db254d94..7c50bdbe0 100644
--- a/src/video_core/engines/kepler_compute.cpp
+++ b/src/video_core/engines/kepler_compute.cpp
@@ -36,8 +36,6 @@ void KeplerCompute::CallMethod(u32 method, u32 method_argument, bool is_last_cal
     }
     case KEPLER_COMPUTE_REG_INDEX(data_upload): {
         upload_state.ProcessData(method_argument, is_last_call);
-        if (is_last_call) {
-        }
         break;
     }
     case KEPLER_COMPUTE_REG_INDEX(launch):
@@ -50,8 +48,15 @@ void KeplerCompute::CallMethod(u32 method, u32 method_argument, bool is_last_cal
 
 void KeplerCompute::CallMultiMethod(u32 method, const u32* base_start, u32 amount,
                                     u32 methods_pending) {
-    for (std::size_t i = 0; i < amount; i++) {
-        CallMethod(method, base_start[i], methods_pending - static_cast<u32>(i) <= 1);
+    switch (method) {
+    case KEPLER_COMPUTE_REG_INDEX(data_upload):
+        upload_state.ProcessData(base_start, static_cast<size_t>(amount));
+        return;
+    default:
+        for (std::size_t i = 0; i < amount; i++) {
+            CallMethod(method, base_start[i], methods_pending - static_cast<u32>(i) <= 1);
+        }
+        break;
     }
 }
 
diff --git a/src/video_core/engines/kepler_memory.cpp b/src/video_core/engines/kepler_memory.cpp
index e2b029542..a3fbab1e5 100644
--- a/src/video_core/engines/kepler_memory.cpp
+++ b/src/video_core/engines/kepler_memory.cpp
@@ -33,8 +33,6 @@ void KeplerMemory::CallMethod(u32 method, u32 method_argument, bool is_last_call
     }
     case KEPLERMEMORY_REG_INDEX(data): {
         upload_state.ProcessData(method_argument, is_last_call);
-        if (is_last_call) {
-        }
         break;
     }
     }
@@ -42,8 +40,15 @@ void KeplerMemory::CallMethod(u32 method, u32 method_argument, bool is_last_call
 
 void KeplerMemory::CallMultiMethod(u32 method, const u32* base_start, u32 amount,
                                    u32 methods_pending) {
-    for (std::size_t i = 0; i < amount; i++) {
-        CallMethod(method, base_start[i], methods_pending - static_cast<u32>(i) <= 1);
+    switch (method) {
+    case KEPLERMEMORY_REG_INDEX(data):
+        upload_state.ProcessData(base_start, static_cast<size_t>(amount));
+        return;
+    default:
+        for (std::size_t i = 0; i < amount; i++) {
+            CallMethod(method, base_start[i], methods_pending - static_cast<u32>(i) <= 1);
+        }
+        break;
     }
 }
 
diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index add1ccebe..632052c53 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -239,8 +239,6 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume
         return upload_state.ProcessExec(regs.exec_upload.linear != 0);
     case MAXWELL3D_REG_INDEX(data_upload):
         upload_state.ProcessData(argument, is_last_call);
-        if (is_last_call) {
-        }
         return;
     case MAXWELL3D_REG_INDEX(fragment_barrier):
         return rasterizer->FragmentBarrier();
@@ -316,6 +314,9 @@ void Maxwell3D::CallMultiMethod(u32 method, const u32* base_start, u32 amount,
     case MAXWELL3D_REG_INDEX(const_buffer.cb_data) + 15:
         ProcessCBMultiData(base_start, amount);
         break;
+    case MAXWELL3D_REG_INDEX(data_upload):
+        upload_state.ProcessData(base_start, static_cast<size_t>(amount));
+        return;
     default:
         for (std::size_t i = 0; i < amount; i++) {
             CallMethod(method, base_start[i], methods_pending - static_cast<u32>(i) <= 1);
diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp
index 0efe58282..a12a95ce2 100644
--- a/src/video_core/engines/maxwell_dma.cpp
+++ b/src/video_core/engines/maxwell_dma.cpp
@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include "common/algorithm.h"
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/microprofile.h"
@@ -54,8 +55,6 @@ void MaxwellDMA::Launch() {
     const LaunchDMA& launch = regs.launch_dma;
     ASSERT(launch.interrupt_type == LaunchDMA::InterruptType::NONE);
     ASSERT(launch.data_transfer_type == LaunchDMA::DataTransferType::NON_PIPELINED);
-    ASSERT(regs.dst_params.origin.x == 0);
-    ASSERT(regs.dst_params.origin.y == 0);
 
     const bool is_src_pitch = launch.src_memory_layout == LaunchDMA::MemoryLayout::PITCH;
     const bool is_dst_pitch = launch.dst_memory_layout == LaunchDMA::MemoryLayout::PITCH;
@@ -121,12 +120,13 @@ void MaxwellDMA::CopyPitchToPitch() {
 
 void MaxwellDMA::CopyBlockLinearToPitch() {
     UNIMPLEMENTED_IF(regs.src_params.block_size.width != 0);
-    UNIMPLEMENTED_IF(regs.src_params.block_size.depth != 0);
     UNIMPLEMENTED_IF(regs.src_params.layer != 0);
 
+    const bool is_remapping = regs.launch_dma.remap_enable != 0;
+
     // Optimized path for micro copies.
     const size_t dst_size = static_cast<size_t>(regs.pitch_out) * regs.line_count;
-    if (dst_size < GOB_SIZE && regs.pitch_out <= GOB_SIZE_X &&
+    if (!is_remapping && dst_size < GOB_SIZE && regs.pitch_out <= GOB_SIZE_X &&
         regs.src_params.height > GOB_SIZE_Y) {
         FastCopyBlockLinearToPitch();
         return;
@@ -134,10 +134,27 @@ void MaxwellDMA::CopyBlockLinearToPitch() {
 
     // Deswizzle the input and copy it over.
     UNIMPLEMENTED_IF(regs.launch_dma.remap_enable != 0);
-    const u32 bytes_per_pixel =
-        regs.launch_dma.remap_enable ? regs.pitch_out / regs.line_length_in : 1;
     const Parameters& src_params = regs.src_params;
-    const u32 width = src_params.width;
+
+    const u32 num_remap_components = regs.remap_const.num_dst_components_minus_one + 1;
+    const u32 remap_components_size = regs.remap_const.component_size_minus_one + 1;
+
+    const u32 base_bpp = !is_remapping ? 1U : num_remap_components * remap_components_size;
+
+    u32 width = src_params.width;
+    u32 x_elements = regs.line_length_in;
+    u32 x_offset = src_params.origin.x;
+    u32 bpp_shift = 0U;
+    if (!is_remapping) {
+        bpp_shift = Common::FoldRight(
+            4U, [](u32 x, u32 y) { return std::min(x, static_cast<u32>(std::countr_zero(y))); },
+            width, x_elements, x_offset, static_cast<u32>(regs.offset_in));
+        width >>= bpp_shift;
+        x_elements >>= bpp_shift;
+        x_offset >>= bpp_shift;
+    }
+
+    const u32 bytes_per_pixel = base_bpp << bpp_shift;
     const u32 height = src_params.height;
     const u32 depth = src_params.depth;
     const u32 block_height = src_params.block_size.height;
@@ -155,30 +172,46 @@ void MaxwellDMA::CopyBlockLinearToPitch() {
     memory_manager.ReadBlock(regs.offset_in, read_buffer.data(), src_size);
     memory_manager.ReadBlock(regs.offset_out, write_buffer.data(), dst_size);
 
-    UnswizzleSubrect(regs.line_length_in, regs.line_count, regs.pitch_out, width, bytes_per_pixel,
-                     block_height, src_params.origin.x, src_params.origin.y, write_buffer.data(),
-                     read_buffer.data());
+    UnswizzleSubrect(write_buffer, read_buffer, bytes_per_pixel, width, height, depth, x_offset,
+                     src_params.origin.y, x_elements, regs.line_count, block_height, block_depth,
+                     regs.pitch_out);
 
     memory_manager.WriteBlock(regs.offset_out, write_buffer.data(), dst_size);
 }
 
 void MaxwellDMA::CopyPitchToBlockLinear() {
     UNIMPLEMENTED_IF_MSG(regs.dst_params.block_size.width != 0, "Block width is not one");
+    UNIMPLEMENTED_IF(regs.dst_params.layer != 0);
     UNIMPLEMENTED_IF(regs.launch_dma.remap_enable != 0);
 
+    const bool is_remapping = regs.launch_dma.remap_enable != 0;
+    const u32 num_remap_components = regs.remap_const.num_dst_components_minus_one + 1;
+    const u32 remap_components_size = regs.remap_const.component_size_minus_one + 1;
+
     const auto& dst_params = regs.dst_params;
-    const u32 bytes_per_pixel =
-        regs.launch_dma.remap_enable ? regs.pitch_in / regs.line_length_in : 1;
-    const u32 width = dst_params.width;
+
+    const u32 base_bpp = !is_remapping ? 1U : num_remap_components * remap_components_size;
+
+    u32 width = dst_params.width;
+    u32 x_elements = regs.line_length_in;
+    u32 x_offset = dst_params.origin.x;
+    u32 bpp_shift = 0U;
+    if (!is_remapping) {
+        bpp_shift = Common::FoldRight(
+            4U, [](u32 x, u32 y) { return std::min(x, static_cast<u32>(std::countr_zero(y))); },
+            width, x_elements, x_offset, static_cast<u32>(regs.offset_out));
+        width >>= bpp_shift;
+        x_elements >>= bpp_shift;
+        x_offset >>= bpp_shift;
+    }
+
+    const u32 bytes_per_pixel = base_bpp << bpp_shift;
     const u32 height = dst_params.height;
     const u32 depth = dst_params.depth;
     const u32 block_height = dst_params.block_size.height;
     const u32 block_depth = dst_params.block_size.depth;
     const size_t dst_size =
         CalculateSize(true, bytes_per_pixel, width, height, depth, block_height, block_depth);
-    const size_t dst_layer_size =
-        CalculateSize(true, bytes_per_pixel, width, height, 1, block_height, block_depth);
-
     const size_t src_size = static_cast<size_t>(regs.pitch_in) * regs.line_count;
 
     if (read_buffer.size() < src_size) {
@@ -188,32 +221,23 @@ void MaxwellDMA::CopyPitchToBlockLinear() {
         write_buffer.resize(dst_size);
     }
 
+    memory_manager.ReadBlock(regs.offset_in, read_buffer.data(), src_size);
     if (Settings::IsGPULevelExtreme()) {
-        memory_manager.ReadBlock(regs.offset_in, read_buffer.data(), src_size);
         memory_manager.ReadBlock(regs.offset_out, write_buffer.data(), dst_size);
     } else {
-        memory_manager.ReadBlockUnsafe(regs.offset_in, read_buffer.data(), src_size);
         memory_manager.ReadBlockUnsafe(regs.offset_out, write_buffer.data(), dst_size);
     }
 
     // If the input is linear and the output is tiled, swizzle the input and copy it over.
-    if (regs.dst_params.block_size.depth > 0) {
-        ASSERT(dst_params.layer == 0);
-        SwizzleSliceToVoxel(regs.line_length_in, regs.line_count, regs.pitch_in, width, height,
-                            bytes_per_pixel, block_height, block_depth, dst_params.origin.x,
-                            dst_params.origin.y, write_buffer.data(), read_buffer.data());
-    } else {
-        SwizzleSubrect(regs.line_length_in, regs.line_count, regs.pitch_in, width, bytes_per_pixel,
-                       write_buffer.data() + dst_layer_size * dst_params.layer, read_buffer.data(),
-                       block_height, dst_params.origin.x, dst_params.origin.y);
-    }
+    SwizzleSubrect(write_buffer, read_buffer, bytes_per_pixel, width, height, depth, x_offset,
+                   dst_params.origin.y, x_elements, regs.line_count, block_height, block_depth,
+                   regs.pitch_in);
 
     memory_manager.WriteBlock(regs.offset_out, write_buffer.data(), dst_size);
 }
 
 void MaxwellDMA::FastCopyBlockLinearToPitch() {
-    const u32 bytes_per_pixel =
-        regs.launch_dma.remap_enable ? regs.pitch_out / regs.line_length_in : 1;
+    const u32 bytes_per_pixel = 1U;
     const size_t src_size = GOB_SIZE;
     const size_t dst_size = static_cast<size_t>(regs.pitch_out) * regs.line_count;
     u32 pos_x = regs.src_params.origin.x;
@@ -239,9 +263,10 @@ void MaxwellDMA::FastCopyBlockLinearToPitch() {
         memory_manager.ReadBlockUnsafe(regs.offset_out, write_buffer.data(), dst_size);
     }
 
-    UnswizzleSubrect(regs.line_length_in, regs.line_count, regs.pitch_out, regs.src_params.width,
-                     bytes_per_pixel, regs.src_params.block_size.height, pos_x, pos_y,
-                     write_buffer.data(), read_buffer.data());
+    UnswizzleSubrect(write_buffer, read_buffer, bytes_per_pixel, regs.src_params.width,
+                     regs.src_params.height, 1, pos_x, pos_y, regs.line_length_in, regs.line_count,
+                     regs.src_params.block_size.height, regs.src_params.block_size.depth,
+                     regs.pitch_out);
 
     memory_manager.WriteBlock(regs.offset_out, write_buffer.data(), dst_size);
 }
diff --git a/src/video_core/engines/maxwell_dma.h b/src/video_core/engines/maxwell_dma.h
index 074bac92c..9c5d567a6 100644
--- a/src/video_core/engines/maxwell_dma.h
+++ b/src/video_core/engines/maxwell_dma.h
@@ -189,10 +189,16 @@ public:
             BitField<4, 3, Swizzle> dst_y;
             BitField<8, 3, Swizzle> dst_z;
             BitField<12, 3, Swizzle> dst_w;
+            BitField<0, 12, u32> dst_components_raw;
             BitField<16, 2, u32> component_size_minus_one;
             BitField<20, 2, u32> num_src_components_minus_one;
             BitField<24, 2, u32> num_dst_components_minus_one;
         };
+
+        Swizzle GetComponent(size_t i) {
+            const u32 raw = dst_components_raw;
+            return static_cast<Swizzle>((raw >> (i * 3)) & 0x7);
+        }
     };
     static_assert(sizeof(RemapConst) == 12);