video_core: Rewrite the texture cache

The current texture cache has several points that hurt maintainability
and performance. It's easy to break unrelated parts of the cache
when doing minor changes. The cache can easily forget valuable
information about the cached textures by CPU writes or simply by its
normal usage.The current texture cache has several points that hurt
maintainability and performance. It's easy to break unrelated parts
of the cache when doing minor changes. The cache can easily forget
valuable information about the cached textures by CPU writes or simply
by its normal usage.

This commit aims to address those issues.

1 files changed, 78 insertions, 171 deletions

diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp
index 16d46a018..9f5181318 100644
--- a/src/video_core/textures/decoders.cpp
+++ b/src/video_core/textures/decoders.cpp
@@ -2,204 +2,111 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <array>
 #include <cmath>
 #include <cstring>
+#include <span>
+#include <utility>
+
 #include "common/alignment.h"
 #include "common/assert.h"
 #include "common/bit_util.h"
+#include "common/div_ceil.h"
 #include "video_core/gpu.h"
 #include "video_core/textures/decoders.h"
 #include "video_core/textures/texture.h"
 
 namespace Tegra::Texture {
-namespace {
 
+namespace {
 /**
- * This table represents the internal swizzle of a gob,
- * in format 16 bytes x 2 sector packing.
+ * This table represents the internal swizzle of a gob, in format 16 bytes x 2 sector packing.
  * Calculates the offset of an (x, y) position within a swizzled texture.
  * Taken from the Tegra X1 Technical Reference Manual. pages 1187-1188
  */
-template <std::size_t N, std::size_t M, u32 Align>
-struct alignas(64) SwizzleTable {
-    static_assert(M * Align == 64, "Swizzle Table does not align to GOB");
-    constexpr SwizzleTable() {
-        for (u32 y = 0; y < N; ++y) {
-            for (u32 x = 0; x < M; ++x) {
-                const u32 x2 = x * Align;
-                values[y][x] = static_cast<u16>(((x2 % 64) / 32) * 256 + ((y % 8) / 2) * 64 +
-                                                ((x2 % 32) / 16) * 32 + (y % 2) * 16 + (x2 % 16));
-            }
+constexpr SwizzleTable MakeSwizzleTableConst() {
+    SwizzleTable table{};
+    for (u32 y = 0; y < table.size(); ++y) {
+        for (u32 x = 0; x < table[0].size(); ++x) {
+            table[y][x] = ((x % 64) / 32) * 256 + ((y % 8) / 2) * 64 + ((x % 32) / 16) * 32 +
+                          (y % 2) * 16 + (x % 16);
         }
     }
-    const std::array<u16, M>& operator[](std::size_t index) const {
-        return values[index];
-    }
-    std::array<std::array<u16, M>, N> values{};
-};
+    return table;
+}
 
-constexpr u32 FAST_SWIZZLE_ALIGN = 16;
+constexpr SwizzleTable SWIZZLE_TABLE = MakeSwizzleTableConst();
 
-constexpr auto LEGACY_SWIZZLE_TABLE = SwizzleTable<GOB_SIZE_X, GOB_SIZE_X, GOB_SIZE_Z>();
-constexpr auto FAST_SWIZZLE_TABLE = SwizzleTable<GOB_SIZE_Y, 4, FAST_SWIZZLE_ALIGN>();
+template <bool TO_LINEAR>
+void Swizzle(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
+             u32 height, u32 depth, u32 block_height, u32 block_depth, u32 stride_alignment) {
+    // The origin of the transformation can be configured here, leave it as zero as the current API
+    // doesn't expose it.
+    static constexpr u32 origin_x = 0;
+    static constexpr u32 origin_y = 0;
+    static constexpr u32 origin_z = 0;
 
-/**
- * This function manages ALL the GOBs(Group of Bytes) Inside a single block.
- * Instead of going gob by gob, we map the coordinates inside a block and manage from
- * those. Block_Width is assumed to be 1.
- */
-void PreciseProcessBlock(u8* const swizzled_data, u8* const unswizzled_data, const bool unswizzle,
-                         const u32 x_start, const u32 y_start, const u32 z_start, const u32 x_end,
-                         const u32 y_end, const u32 z_end, const u32 tile_offset,
-                         const u32 xy_block_size, const u32 layer_z, const u32 stride_x,
-                         const u32 bytes_per_pixel, const u32 out_bytes_per_pixel) {
-    std::array<u8*, 2> data_ptrs;
-    u32 z_address = tile_offset;
-
-    for (u32 z = z_start; z < z_end; z++) {
-        u32 y_address = z_address;
-        u32 pixel_base = layer_z * z + y_start * stride_x;
-        for (u32 y = y_start; y < y_end; y++) {
-            const auto& table = LEGACY_SWIZZLE_TABLE[y % GOB_SIZE_Y];
-            for (u32 x = x_start; x < x_end; x++) {
-                const u32 swizzle_offset{y_address + table[x * bytes_per_pixel % GOB_SIZE_X]};
-                const u32 pixel_index{x * out_bytes_per_pixel + pixel_base};
-                data_ptrs[unswizzle] = swizzled_data + swizzle_offset;
-                data_ptrs[!unswizzle] = unswizzled_data + pixel_index;
-                std::memcpy(data_ptrs[0], data_ptrs[1], bytes_per_pixel);
-            }
-            pixel_base += stride_x;
-            if ((y + 1) % GOB_SIZE_Y == 0)
-                y_address += GOB_SIZE;
-        }
-        z_address += xy_block_size;
-    }
-}
+    // We can configure here a custom pitch
+    // As it's not exposed 'width * bpp' will be the expected pitch.
+    const u32 pitch = width * bytes_per_pixel;
+    const u32 stride = Common::AlignBits(width, stride_alignment) * bytes_per_pixel;
 
-/**
- * This function manages ALL the GOBs(Group of Bytes) Inside a single block.
- * Instead of going gob by gob, we map the coordinates inside a block and manage from
- * those. Block_Width is assumed to be 1.
- */
-void FastProcessBlock(u8* const swizzled_data, u8* const unswizzled_data, const bool unswizzle,
-                      const u32 x_start, const u32 y_start, const u32 z_start, const u32 x_end,
-                      const u32 y_end, const u32 z_end, const u32 tile_offset,
-                      const u32 xy_block_size, const u32 layer_z, const u32 stride_x,
-                      const u32 bytes_per_pixel, const u32 out_bytes_per_pixel) {
-    std::array<u8*, 2> data_ptrs;
-    u32 z_address = tile_offset;
-    const u32 x_startb = x_start * bytes_per_pixel;
-    const u32 x_endb = x_end * bytes_per_pixel;
-
-    for (u32 z = z_start; z < z_end; z++) {
-        u32 y_address = z_address;
-        u32 pixel_base = layer_z * z + y_start * stride_x;
-        for (u32 y = y_start; y < y_end; y++) {
-            const auto& table = FAST_SWIZZLE_TABLE[y % GOB_SIZE_Y];
-            for (u32 xb = x_startb; xb < x_endb; xb += FAST_SWIZZLE_ALIGN) {
-                const u32 swizzle_offset{y_address + table[(xb / FAST_SWIZZLE_ALIGN) % 4]};
-                const u32 out_x = xb * out_bytes_per_pixel / bytes_per_pixel;
-                const u32 pixel_index{out_x + pixel_base};
-                data_ptrs[unswizzle ? 1 : 0] = swizzled_data + swizzle_offset;
-                data_ptrs[unswizzle ? 0 : 1] = unswizzled_data + pixel_index;
-                std::memcpy(data_ptrs[0], data_ptrs[1], FAST_SWIZZLE_ALIGN);
-            }
-            pixel_base += stride_x;
-            if ((y + 1) % GOB_SIZE_Y == 0)
-                y_address += GOB_SIZE;
-        }
-        z_address += xy_block_size;
-    }
-}
+    const u32 gobs_in_x = Common::DivCeilLog2(stride, GOB_SIZE_X_SHIFT);
+    const u32 block_size = gobs_in_x << (GOB_SIZE_SHIFT + block_height + block_depth);
+    const u32 slice_size =
+        Common::DivCeilLog2(height, block_height + GOB_SIZE_Y_SHIFT) * block_size;
 
-/**
- * This function unswizzles or swizzles a texture by mapping Linear to BlockLinear Textue.
- * The body of this function takes care of splitting the swizzled texture into blocks,
- * and managing the extents of it. Once all the parameters of a single block are obtained,
- * the function calls 'ProcessBlock' to process that particular Block.
- *
- * Documentation for the memory layout and decoding can be found at:
- *  https://envytools.readthedocs.io/en/latest/hw/memory/g80-surface.html#blocklinear-surfaces
- */
-template <bool fast>
-void SwizzledData(u8* const swizzled_data, u8* const unswizzled_data, const bool unswizzle,
-                  const u32 width, const u32 height, const u32 depth, const u32 bytes_per_pixel,
-                  const u32 out_bytes_per_pixel, const u32 block_height, const u32 block_depth,
-                  const u32 width_spacing) {
-    auto div_ceil = [](const u32 x, const u32 y) { return ((x + y - 1) / y); };
-    const u32 stride_x = width * out_bytes_per_pixel;
-    const u32 layer_z = height * stride_x;
-    const u32 gob_elements_x = GOB_SIZE_X / bytes_per_pixel;
-    constexpr u32 gob_elements_y = GOB_SIZE_Y;
-    constexpr u32 gob_elements_z = GOB_SIZE_Z;
-    const u32 block_x_elements = gob_elements_x;
-    const u32 block_y_elements = gob_elements_y * block_height;
-    const u32 block_z_elements = gob_elements_z * block_depth;
-    const u32 aligned_width = Common::AlignUp(width, gob_elements_x * width_spacing);
-    const u32 blocks_on_x = div_ceil(aligned_width, block_x_elements);
-    const u32 blocks_on_y = div_ceil(height, block_y_elements);
-    const u32 blocks_on_z = div_ceil(depth, block_z_elements);
-    const u32 xy_block_size = GOB_SIZE * block_height;
-    const u32 block_size = xy_block_size * block_depth;
-    u32 tile_offset = 0;
-    for (u32 zb = 0; zb < blocks_on_z; zb++) {
-        const u32 z_start = zb * block_z_elements;
-        const u32 z_end = std::min(depth, z_start + block_z_elements);
-        for (u32 yb = 0; yb < blocks_on_y; yb++) {
-            const u32 y_start = yb * block_y_elements;
-            const u32 y_end = std::min(height, y_start + block_y_elements);
-            for (u32 xb = 0; xb < blocks_on_x; xb++) {
-                const u32 x_start = xb * block_x_elements;
-                const u32 x_end = std::min(width, x_start + block_x_elements);
-                if constexpr (fast) {
-                    FastProcessBlock(swizzled_data, unswizzled_data, unswizzle, x_start, y_start,
-                                     z_start, x_end, y_end, z_end, tile_offset, xy_block_size,
-                                     layer_z, stride_x, bytes_per_pixel, out_bytes_per_pixel);
-                } else {
-                    PreciseProcessBlock(swizzled_data, unswizzled_data, unswizzle, x_start, y_start,
-                                        z_start, x_end, y_end, z_end, tile_offset, xy_block_size,
-                                        layer_z, stride_x, bytes_per_pixel, out_bytes_per_pixel);
-                }
-                tile_offset += block_size;
+    const u32 block_height_mask = (1U << block_height) - 1;
+    const u32 block_depth_mask = (1U << block_depth) - 1;
+    const u32 x_shift = GOB_SIZE_SHIFT + block_height + block_depth;
+
+    for (u32 slice = 0; slice < depth; ++slice) {
+        const u32 z = slice + origin_z;
+        const u32 offset_z = (z >> block_depth) * slice_size +
+                             ((z & block_depth_mask) << (GOB_SIZE_SHIFT + block_height));
+        for (u32 line = 0; line < height; ++line) {
+            const u32 y = line + origin_y;
+            const auto& table = SWIZZLE_TABLE[y % GOB_SIZE_Y];
+
+            const u32 block_y = y >> GOB_SIZE_Y_SHIFT;
+            const u32 offset_y = (block_y >> block_height) * block_size +
+                                 ((block_y & block_height_mask) << GOB_SIZE_SHIFT);
+
+            for (u32 column = 0; column < width; ++column) {
+                const u32 x = (column + origin_x) * bytes_per_pixel;
+                const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift;
+
+                const u32 base_swizzled_offset = offset_z + offset_y + offset_x;
+                const u32 swizzled_offset = base_swizzled_offset + table[x % GOB_SIZE_X];
+
+                const u32 unswizzled_offset =
+                    slice * pitch * height + line * pitch + column * bytes_per_pixel;
+
+                u8* const dst = &output[TO_LINEAR ? swizzled_offset : unswizzled_offset];
+                const u8* const src = &input[TO_LINEAR ? unswizzled_offset : swizzled_offset];
+                std::memcpy(dst, src, bytes_per_pixel);
             }
         }
     }
 }
-
 } // Anonymous namespace
 
-void CopySwizzledData(u32 width, u32 height, u32 depth, u32 bytes_per_pixel,
-                      u32 out_bytes_per_pixel, u8* const swizzled_data, u8* const unswizzled_data,
-                      bool unswizzle, u32 block_height, u32 block_depth, u32 width_spacing) {
-    const u32 block_height_size{1U << block_height};
-    const u32 block_depth_size{1U << block_depth};
-    if (bytes_per_pixel % 3 != 0 && (width * bytes_per_pixel) % FAST_SWIZZLE_ALIGN == 0) {
-        SwizzledData<true>(swizzled_data, unswizzled_data, unswizzle, width, height, depth,
-                           bytes_per_pixel, out_bytes_per_pixel, block_height_size,
-                           block_depth_size, width_spacing);
-    } else {
-        SwizzledData<false>(swizzled_data, unswizzled_data, unswizzle, width, height, depth,
-                            bytes_per_pixel, out_bytes_per_pixel, block_height_size,
-                            block_depth_size, width_spacing);
-    }
+SwizzleTable MakeSwizzleTable() {
+    return SWIZZLE_TABLE;
 }
 
-void UnswizzleTexture(u8* const unswizzled_data, u8* address, u32 tile_size_x, u32 tile_size_y,
-                      u32 bytes_per_pixel, u32 width, u32 height, u32 depth, u32 block_height,
-                      u32 block_depth, u32 width_spacing) {
-    CopySwizzledData((width + tile_size_x - 1) / tile_size_x,
-                     (height + tile_size_y - 1) / tile_size_y, depth, bytes_per_pixel,
-                     bytes_per_pixel, address, unswizzled_data, true, block_height, block_depth,
-                     width_spacing);
+void UnswizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel,
+                      u32 width, u32 height, u32 depth, u32 block_height, u32 block_depth,
+                      u32 stride_alignment) {
+    Swizzle<false>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
+                   stride_alignment);
 }
 
-std::vector<u8> UnswizzleTexture(u8* address, u32 tile_size_x, u32 tile_size_y, u32 bytes_per_pixel,
-                                 u32 width, u32 height, u32 depth, u32 block_height,
-                                 u32 block_depth, u32 width_spacing) {
-    std::vector<u8> unswizzled_data(width * height * depth * bytes_per_pixel);
-    UnswizzleTexture(unswizzled_data.data(), address, tile_size_x, tile_size_y, bytes_per_pixel,
-                     width, height, depth, block_height, block_depth, width_spacing);
-    return unswizzled_data;
+void SwizzleTexture(std::span<u8> output, std::span<const u8> input, u32 bytes_per_pixel, u32 width,
+                    u32 height, u32 depth, u32 block_height, u32 block_depth,
+                    u32 stride_alignment) {
+    Swizzle<true>(output, input, bytes_per_pixel, width, height, depth, block_height, block_depth,
+                  stride_alignment);
 }
 
 void SwizzleSubrect(u32 subrect_width, u32 subrect_height, u32 source_pitch, u32 swizzled_width,
@@ -213,7 +120,7 @@ void SwizzleSubrect(u32 subrect_width, u32 subrect_height, u32 source_pitch, u32
         const u32 gob_address_y =
             (dst_y / (GOB_SIZE_Y * block_height)) * GOB_SIZE * block_height * image_width_in_gobs +
             ((dst_y % (GOB_SIZE_Y * block_height)) / GOB_SIZE_Y) * GOB_SIZE;
-        const auto& table = LEGACY_SWIZZLE_TABLE[dst_y % GOB_SIZE_Y];
+        const auto& table = SWIZZLE_TABLE[dst_y % GOB_SIZE_Y];
         for (u32 x = 0; x < subrect_width; ++x) {
             const u32 dst_x = x + offset_x;
             const u32 gob_address =
@@ -235,11 +142,11 @@ void UnswizzleSubrect(u32 line_length_in, u32 line_count, u32 pitch, u32 width,
     const u32 block_size = gobs_in_x << (GOB_SIZE_SHIFT + block_height);
 
     const u32 block_height_mask = (1U << block_height) - 1;
-    const u32 x_shift = static_cast<u32>(GOB_SIZE_SHIFT) + block_height;
+    const u32 x_shift = GOB_SIZE_SHIFT + block_height;
 
     for (u32 line = 0; line < line_count; ++line) {
         const u32 src_y = line + origin_y;
-        const auto& table = LEGACY_SWIZZLE_TABLE[src_y % GOB_SIZE_Y];
+        const auto& table = SWIZZLE_TABLE[src_y % GOB_SIZE_Y];
 
         const u32 block_y = src_y >> GOB_SIZE_Y_SHIFT;
         const u32 src_offset_y = (block_y >> block_height) * block_size +
@@ -270,7 +177,7 @@ void SwizzleSliceToVoxel(u32 line_length_in, u32 line_count, u32 pitch, u32 widt
     const u32 x_shift = static_cast<u32>(GOB_SIZE_SHIFT) + block_height + block_depth;
 
     for (u32 line = 0; line < line_count; ++line) {
-        const auto& table = LEGACY_SWIZZLE_TABLE[line % GOB_SIZE_Y];
+        const auto& table = SWIZZLE_TABLE[line % GOB_SIZE_Y];
         const u32 block_y = line / GOB_SIZE_Y;
         const u32 dst_offset_y =
             (block_y >> block_height) * block_size + (block_y & block_height_mask) * GOB_SIZE;
@@ -293,7 +200,7 @@ void SwizzleKepler(const u32 width, const u32 height, const u32 dst_x, const u32
         const std::size_t gob_address_y =
             (y / (GOB_SIZE_Y * block_height)) * GOB_SIZE * block_height * image_width_in_gobs +
             ((y % (GOB_SIZE_Y * block_height)) / GOB_SIZE_Y) * GOB_SIZE;
-        const auto& table = LEGACY_SWIZZLE_TABLE[y % GOB_SIZE_Y];
+        const auto& table = SWIZZLE_TABLE[y % GOB_SIZE_Y];
         for (std::size_t x = dst_x; x < width && count < copy_size; ++x) {
             const std::size_t gob_address =
                 gob_address_y + (x / GOB_SIZE_X) * GOB_SIZE * block_height;