diff options
| -rw-r--r-- | src/tests/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/tests/video_core/buffer_base.cpp | 473 | ||||
| -rw-r--r-- | src/video_core/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/video_core/buffer_cache/buffer_base.h | 495 |
4 files changed, 970 insertions, 0 deletions
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 33fa89583..6a5c18945 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt | |||
| @@ -5,6 +5,7 @@ add_executable(tests | |||
| 5 | common/ring_buffer.cpp | 5 | common/ring_buffer.cpp |
| 6 | core/core_timing.cpp | 6 | core/core_timing.cpp |
| 7 | tests.cpp | 7 | tests.cpp |
| 8 | video_core/buffer_base.cpp | ||
| 8 | ) | 9 | ) |
| 9 | 10 | ||
| 10 | create_target_directory_groups(tests) | 11 | create_target_directory_groups(tests) |
diff --git a/src/tests/video_core/buffer_base.cpp b/src/tests/video_core/buffer_base.cpp new file mode 100644 index 000000000..651633e9e --- /dev/null +++ b/src/tests/video_core/buffer_base.cpp | |||
| @@ -0,0 +1,473 @@ | |||
| 1 | // Copyright 2020 yuzu Emulator Project | ||
| 2 | // Licensed under GPLv2 or any later version | ||
| 3 | // Refer to the license.txt file included. | ||
| 4 | |||
| 5 | #include <stdexcept> | ||
| 6 | #include <unordered_map> | ||
| 7 | |||
| 8 | #include <catch2/catch.hpp> | ||
| 9 | |||
| 10 | #include "common/alignment.h" | ||
| 11 | #include "common/common_types.h" | ||
| 12 | #include "video_core/buffer_cache/buffer_base.h" | ||
| 13 | |||
| 14 | namespace { | ||
| 15 | using VideoCommon::BufferBase; | ||
| 16 | using Range = std::pair<u64, u64>; | ||
| 17 | |||
| 18 | constexpr u64 PAGE = 4096; | ||
| 19 | constexpr u64 WORD = 4096 * 64; | ||
| 20 | |||
| 21 | constexpr VAddr c = 0x1328914000; | ||
| 22 | |||
| 23 | class RasterizerInterface { | ||
| 24 | public: | ||
| 25 | void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) { | ||
| 26 | const u64 page_start{addr >> Core::Memory::PAGE_BITS}; | ||
| 27 | const u64 page_end{(addr + size + Core::Memory::PAGE_SIZE - 1) >> Core::Memory::PAGE_BITS}; | ||
| 28 | for (u64 page = page_start; page < page_end; ++page) { | ||
| 29 | int& value = page_table[page]; | ||
| 30 | value += delta; | ||
| 31 | if (value < 0) { | ||
| 32 | throw std::logic_error{"negative page"}; | ||
| 33 | } | ||
| 34 | if (value == 0) { | ||
| 35 | page_table.erase(page); | ||
| 36 | } | ||
| 37 | } | ||
| 38 | } | ||
| 39 | |||
| 40 | [[nodiscard]] int Count(VAddr addr) const noexcept { | ||
| 41 | const auto it = page_table.find(addr >> Core::Memory::PAGE_BITS); | ||
| 42 | return it == page_table.end() ? 0 : it->second; | ||
| 43 | } | ||
| 44 | |||
| 45 | [[nodiscard]] unsigned Count() const noexcept { | ||
| 46 | unsigned count = 0; | ||
| 47 | for (const auto [index, value] : page_table) { | ||
| 48 | count += value; | ||
| 49 | } | ||
| 50 | return count; | ||
| 51 | } | ||
| 52 | |||
| 53 | private: | ||
| 54 | std::unordered_map<u64, int> page_table; | ||
| 55 | }; | ||
| 56 | } // Anonymous namespace | ||
| 57 | |||
| 58 | TEST_CASE("BufferBase: Small buffer", "[video_core]") { | ||
| 59 | RasterizerInterface rasterizer; | ||
| 60 | BufferBase buffer(rasterizer, c, WORD); | ||
| 61 | REQUIRE(rasterizer.Count() == 0); | ||
| 62 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 63 | REQUIRE(rasterizer.Count() == WORD / PAGE); | ||
| 64 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{0, 0}); | ||
| 65 | |||
| 66 | buffer.MarkRegionAsCpuModified(c + PAGE, 1); | ||
| 67 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{PAGE * 1, PAGE * 2}); | ||
| 68 | } | ||
| 69 | |||
| 70 | TEST_CASE("BufferBase: Large buffer", "[video_core]") { | ||
| 71 | RasterizerInterface rasterizer; | ||
| 72 | BufferBase buffer(rasterizer, c, WORD * 32); | ||
| 73 | buffer.UnmarkRegionAsCpuModified(c, WORD * 32); | ||
| 74 | buffer.MarkRegionAsCpuModified(c + 4096, WORD * 4); | ||
| 75 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD + PAGE * 2) == Range{PAGE, WORD + PAGE * 2}); | ||
| 76 | REQUIRE(buffer.ModifiedCpuRegion(c + PAGE * 2, PAGE * 6) == Range{PAGE * 2, PAGE * 8}); | ||
| 77 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 4 + PAGE}); | ||
| 78 | REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 4, PAGE) == Range{WORD * 4, WORD * 4 + PAGE}); | ||
| 79 | REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 3 + PAGE * 63, PAGE) == | ||
| 80 | Range{WORD * 3 + PAGE * 63, WORD * 4}); | ||
| 81 | |||
| 82 | buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 6, PAGE); | ||
| 83 | buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE); | ||
| 84 | REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) == | ||
| 85 | Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 9}); | ||
| 86 | |||
| 87 | buffer.UnmarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE); | ||
| 88 | REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) == | ||
| 89 | Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 7}); | ||
| 90 | |||
| 91 | buffer.MarkRegionAsCpuModified(c + PAGE, WORD * 31 + PAGE * 63); | ||
| 92 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 32}); | ||
| 93 | |||
| 94 | buffer.UnmarkRegionAsCpuModified(c + PAGE * 4, PAGE); | ||
| 95 | buffer.UnmarkRegionAsCpuModified(c + PAGE * 6, PAGE); | ||
| 96 | |||
| 97 | buffer.UnmarkRegionAsCpuModified(c, WORD * 32); | ||
| 98 | REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{0, 0}); | ||
| 99 | } | ||
| 100 | |||
| 101 | TEST_CASE("BufferBase: Rasterizer counting", "[video_core]") { | ||
| 102 | RasterizerInterface rasterizer; | ||
| 103 | BufferBase buffer(rasterizer, c, PAGE * 2); | ||
| 104 | REQUIRE(rasterizer.Count() == 0); | ||
| 105 | buffer.UnmarkRegionAsCpuModified(c, PAGE); | ||
| 106 | REQUIRE(rasterizer.Count() == 1); | ||
| 107 | buffer.MarkRegionAsCpuModified(c, PAGE * 2); | ||
| 108 | REQUIRE(rasterizer.Count() == 0); | ||
| 109 | buffer.UnmarkRegionAsCpuModified(c, PAGE); | ||
| 110 | buffer.UnmarkRegionAsCpuModified(c + PAGE, PAGE); | ||
| 111 | REQUIRE(rasterizer.Count() == 2); | ||
| 112 | buffer.MarkRegionAsCpuModified(c, PAGE * 2); | ||
| 113 | REQUIRE(rasterizer.Count() == 0); | ||
| 114 | } | ||
| 115 | |||
| 116 | TEST_CASE("BufferBase: Basic range", "[video_core]") { | ||
| 117 | RasterizerInterface rasterizer; | ||
| 118 | BufferBase buffer(rasterizer, c, WORD); | ||
| 119 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 120 | buffer.MarkRegionAsCpuModified(c, PAGE); | ||
| 121 | int num = 0; | ||
| 122 | buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { | ||
| 123 | REQUIRE(offset == 0U); | ||
| 124 | REQUIRE(size == PAGE); | ||
| 125 | ++num; | ||
| 126 | }); | ||
| 127 | REQUIRE(num == 1U); | ||
| 128 | } | ||
| 129 | |||
| 130 | TEST_CASE("BufferBase: Border upload", "[video_core]") { | ||
| 131 | RasterizerInterface rasterizer; | ||
| 132 | BufferBase buffer(rasterizer, c, WORD * 2); | ||
| 133 | buffer.UnmarkRegionAsCpuModified(c, WORD * 2); | ||
| 134 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 135 | buffer.ForEachUploadRange(c, WORD * 2, [](u64 offset, u64 size) { | ||
| 136 | REQUIRE(offset == WORD - PAGE); | ||
| 137 | REQUIRE(size == PAGE * 2); | ||
| 138 | }); | ||
| 139 | } | ||
| 140 | |||
| 141 | TEST_CASE("BufferBase: Border upload range", "[video_core]") { | ||
| 142 | RasterizerInterface rasterizer; | ||
| 143 | BufferBase buffer(rasterizer, c, WORD * 2); | ||
| 144 | buffer.UnmarkRegionAsCpuModified(c, WORD * 2); | ||
| 145 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 146 | buffer.ForEachUploadRange(c + WORD - PAGE, PAGE * 2, [](u64 offset, u64 size) { | ||
| 147 | REQUIRE(offset == WORD - PAGE); | ||
| 148 | REQUIRE(size == PAGE * 2); | ||
| 149 | }); | ||
| 150 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 151 | buffer.ForEachUploadRange(c + WORD - PAGE, PAGE, [](u64 offset, u64 size) { | ||
| 152 | REQUIRE(offset == WORD - PAGE); | ||
| 153 | REQUIRE(size == PAGE); | ||
| 154 | }); | ||
| 155 | buffer.ForEachUploadRange(c + WORD, PAGE, [](u64 offset, u64 size) { | ||
| 156 | REQUIRE(offset == WORD); | ||
| 157 | REQUIRE(size == PAGE); | ||
| 158 | }); | ||
| 159 | } | ||
| 160 | |||
| 161 | TEST_CASE("BufferBase: Border upload partial range", "[video_core]") { | ||
| 162 | RasterizerInterface rasterizer; | ||
| 163 | BufferBase buffer(rasterizer, c, WORD * 2); | ||
| 164 | buffer.UnmarkRegionAsCpuModified(c, WORD * 2); | ||
| 165 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 166 | buffer.ForEachUploadRange(c + WORD - 1, 2, [](u64 offset, u64 size) { | ||
| 167 | REQUIRE(offset == WORD - PAGE); | ||
| 168 | REQUIRE(size == PAGE * 2); | ||
| 169 | }); | ||
| 170 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 171 | buffer.ForEachUploadRange(c + WORD - 1, 1, [](u64 offset, u64 size) { | ||
| 172 | REQUIRE(offset == WORD - PAGE); | ||
| 173 | REQUIRE(size == PAGE); | ||
| 174 | }); | ||
| 175 | buffer.ForEachUploadRange(c + WORD + 50, 1, [](u64 offset, u64 size) { | ||
| 176 | REQUIRE(offset == WORD); | ||
| 177 | REQUIRE(size == PAGE); | ||
| 178 | }); | ||
| 179 | } | ||
| 180 | |||
| 181 | TEST_CASE("BufferBase: Partial word uploads", "[video_core]") { | ||
| 182 | RasterizerInterface rasterizer; | ||
| 183 | BufferBase buffer(rasterizer, c, 0x9d000); | ||
| 184 | int num = 0; | ||
| 185 | buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { | ||
| 186 | REQUIRE(offset == 0U); | ||
| 187 | REQUIRE(size == WORD); | ||
| 188 | ++num; | ||
| 189 | }); | ||
| 190 | REQUIRE(num == 1); | ||
| 191 | buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { | ||
| 192 | REQUIRE(offset == WORD); | ||
| 193 | REQUIRE(size == WORD); | ||
| 194 | ++num; | ||
| 195 | }); | ||
| 196 | REQUIRE(num == 2); | ||
| 197 | buffer.ForEachUploadRange(c + 0x79000, 0x24000, [&](u64 offset, u64 size) { | ||
| 198 | REQUIRE(offset == WORD * 2); | ||
| 199 | REQUIRE(size == PAGE * 0x1d); | ||
| 200 | ++num; | ||
| 201 | }); | ||
| 202 | REQUIRE(num == 3); | ||
| 203 | } | ||
| 204 | |||
| 205 | TEST_CASE("BufferBase: Partial page upload", "[video_core]") { | ||
| 206 | RasterizerInterface rasterizer; | ||
| 207 | BufferBase buffer(rasterizer, c, WORD); | ||
| 208 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 209 | int num = 0; | ||
| 210 | buffer.MarkRegionAsCpuModified(c + PAGE * 2, PAGE); | ||
| 211 | buffer.MarkRegionAsCpuModified(c + PAGE * 9, PAGE); | ||
| 212 | buffer.ForEachUploadRange(c, PAGE * 3, [&](u64 offset, u64 size) { | ||
| 213 | REQUIRE(offset == PAGE * 2); | ||
| 214 | REQUIRE(size == PAGE); | ||
| 215 | ++num; | ||
| 216 | }); | ||
| 217 | REQUIRE(num == 1); | ||
| 218 | buffer.ForEachUploadRange(c + PAGE * 7, PAGE * 3, [&](u64 offset, u64 size) { | ||
| 219 | REQUIRE(offset == PAGE * 9); | ||
| 220 | REQUIRE(size == PAGE); | ||
| 221 | ++num; | ||
| 222 | }); | ||
| 223 | REQUIRE(num == 2); | ||
| 224 | } | ||
| 225 | |||
| 226 | TEST_CASE("BufferBase: Partial page upload with multiple words on the right") { | ||
| 227 | RasterizerInterface rasterizer; | ||
| 228 | BufferBase buffer(rasterizer, c, WORD * 8); | ||
| 229 | buffer.UnmarkRegionAsCpuModified(c, WORD * 8); | ||
| 230 | buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7); | ||
| 231 | int num = 0; | ||
| 232 | buffer.ForEachUploadRange(c + PAGE * 10, WORD * 7, [&](u64 offset, u64 size) { | ||
| 233 | REQUIRE(offset == PAGE * 13); | ||
| 234 | REQUIRE(size == WORD * 7 - PAGE * 3); | ||
| 235 | ++num; | ||
| 236 | }); | ||
| 237 | REQUIRE(num == 1); | ||
| 238 | buffer.ForEachUploadRange(c + PAGE, WORD * 8, [&](u64 offset, u64 size) { | ||
| 239 | REQUIRE(offset == WORD * 7 + PAGE * 10); | ||
| 240 | REQUIRE(size == PAGE * 3); | ||
| 241 | ++num; | ||
| 242 | }); | ||
| 243 | REQUIRE(num == 2); | ||
| 244 | } | ||
| 245 | |||
| 246 | TEST_CASE("BufferBase: Partial page upload with multiple words on the left", "[video_core]") { | ||
| 247 | RasterizerInterface rasterizer; | ||
| 248 | BufferBase buffer(rasterizer, c, WORD * 8); | ||
| 249 | buffer.UnmarkRegionAsCpuModified(c, WORD * 8); | ||
| 250 | buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7); | ||
| 251 | int num = 0; | ||
| 252 | buffer.ForEachUploadRange(c + PAGE * 16, WORD * 7, [&](u64 offset, u64 size) { | ||
| 253 | REQUIRE(offset == PAGE * 16); | ||
| 254 | REQUIRE(size == WORD * 7 - PAGE * 3); | ||
| 255 | ++num; | ||
| 256 | }); | ||
| 257 | REQUIRE(num == 1); | ||
| 258 | buffer.ForEachUploadRange(c + PAGE, WORD, [&](u64 offset, u64 size) { | ||
| 259 | REQUIRE(offset == PAGE * 13); | ||
| 260 | REQUIRE(size == PAGE * 3); | ||
| 261 | ++num; | ||
| 262 | }); | ||
| 263 | REQUIRE(num == 2); | ||
| 264 | } | ||
| 265 | |||
| 266 | TEST_CASE("BufferBase: Partial page upload with multiple words in the middle", "[video_core]") { | ||
| 267 | RasterizerInterface rasterizer; | ||
| 268 | BufferBase buffer(rasterizer, c, WORD * 8); | ||
| 269 | buffer.UnmarkRegionAsCpuModified(c, WORD * 8); | ||
| 270 | buffer.MarkRegionAsCpuModified(c + PAGE * 13, PAGE * 140); | ||
| 271 | int num = 0; | ||
| 272 | buffer.ForEachUploadRange(c + PAGE * 16, WORD, [&](u64 offset, u64 size) { | ||
| 273 | REQUIRE(offset == PAGE * 16); | ||
| 274 | REQUIRE(size == WORD); | ||
| 275 | ++num; | ||
| 276 | }); | ||
| 277 | REQUIRE(num == 1); | ||
| 278 | buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { | ||
| 279 | REQUIRE(offset == PAGE * 13); | ||
| 280 | REQUIRE(size == PAGE * 3); | ||
| 281 | ++num; | ||
| 282 | }); | ||
| 283 | REQUIRE(num == 2); | ||
| 284 | buffer.ForEachUploadRange(c, WORD * 8, [&](u64 offset, u64 size) { | ||
| 285 | REQUIRE(offset == WORD + PAGE * 16); | ||
| 286 | REQUIRE(size == PAGE * 73); | ||
| 287 | ++num; | ||
| 288 | }); | ||
| 289 | REQUIRE(num == 3); | ||
| 290 | } | ||
| 291 | |||
| 292 | TEST_CASE("BufferBase: Empty right bits", "[video_core]") { | ||
| 293 | RasterizerInterface rasterizer; | ||
| 294 | BufferBase buffer(rasterizer, c, WORD * 2048); | ||
| 295 | buffer.UnmarkRegionAsCpuModified(c, WORD * 2048); | ||
| 296 | buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2); | ||
| 297 | buffer.ForEachUploadRange(c, WORD * 2048, [](u64 offset, u64 size) { | ||
| 298 | REQUIRE(offset == WORD - PAGE); | ||
| 299 | REQUIRE(size == PAGE * 2); | ||
| 300 | }); | ||
| 301 | } | ||
| 302 | |||
| 303 | TEST_CASE("BufferBase: Out of bound ranges 1", "[video_core]") { | ||
| 304 | RasterizerInterface rasterizer; | ||
| 305 | BufferBase buffer(rasterizer, c, WORD); | ||
| 306 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 307 | buffer.MarkRegionAsCpuModified(c, PAGE); | ||
| 308 | int num = 0; | ||
| 309 | buffer.ForEachUploadRange(c - WORD, WORD, [&](u64 offset, u64 size) { ++num; }); | ||
| 310 | buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { ++num; }); | ||
| 311 | buffer.ForEachUploadRange(c - PAGE, PAGE, [&](u64 offset, u64 size) { ++num; }); | ||
| 312 | REQUIRE(num == 0); | ||
| 313 | buffer.ForEachUploadRange(c - PAGE, PAGE * 2, [&](u64 offset, u64 size) { ++num; }); | ||
| 314 | REQUIRE(num == 1); | ||
| 315 | buffer.MarkRegionAsCpuModified(c, WORD); | ||
| 316 | REQUIRE(rasterizer.Count() == 0); | ||
| 317 | } | ||
| 318 | |||
| 319 | TEST_CASE("BufferBase: Out of bound ranges 2", "[video_core]") { | ||
| 320 | RasterizerInterface rasterizer; | ||
| 321 | BufferBase buffer(rasterizer, c, 0x22000); | ||
| 322 | REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x22000, PAGE)); | ||
| 323 | REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x28000, PAGE)); | ||
| 324 | REQUIRE(rasterizer.Count() == 0); | ||
| 325 | REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x21100, PAGE - 0x100)); | ||
| 326 | REQUIRE(rasterizer.Count() == 1); | ||
| 327 | REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c - 0x1000, PAGE * 2)); | ||
| 328 | buffer.UnmarkRegionAsCpuModified(c - 0x3000, PAGE * 2); | ||
| 329 | buffer.UnmarkRegionAsCpuModified(c - 0x2000, PAGE * 2); | ||
| 330 | REQUIRE(rasterizer.Count() == 2); | ||
| 331 | } | ||
| 332 | |||
| 333 | TEST_CASE("BufferBase: Out of bound ranges 3", "[video_core]") { | ||
| 334 | RasterizerInterface rasterizer; | ||
| 335 | BufferBase buffer(rasterizer, c, 0x310720); | ||
| 336 | buffer.UnmarkRegionAsCpuModified(c, 0x310720); | ||
| 337 | REQUIRE(rasterizer.Count(c) == 1); | ||
| 338 | REQUIRE(rasterizer.Count(c + PAGE) == 1); | ||
| 339 | REQUIRE(rasterizer.Count(c + WORD) == 1); | ||
| 340 | REQUIRE(rasterizer.Count(c + WORD + PAGE) == 1); | ||
| 341 | } | ||
| 342 | |||
| 343 | TEST_CASE("BufferBase: Sparse regions 1", "[video_core]") { | ||
| 344 | RasterizerInterface rasterizer; | ||
| 345 | BufferBase buffer(rasterizer, c, WORD); | ||
| 346 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 347 | buffer.MarkRegionAsCpuModified(c + PAGE * 1, PAGE); | ||
| 348 | buffer.MarkRegionAsCpuModified(c + PAGE * 3, PAGE * 4); | ||
| 349 | buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable { | ||
| 350 | static constexpr std::array<u64, 2> offsets{PAGE, PAGE * 3}; | ||
| 351 | static constexpr std::array<u64, 2> sizes{PAGE, PAGE * 4}; | ||
| 352 | REQUIRE(offset == offsets.at(i)); | ||
| 353 | REQUIRE(size == sizes.at(i)); | ||
| 354 | ++i; | ||
| 355 | }); | ||
| 356 | } | ||
| 357 | |||
| 358 | TEST_CASE("BufferBase: Sparse regions 2", "[video_core]") { | ||
| 359 | RasterizerInterface rasterizer; | ||
| 360 | BufferBase buffer(rasterizer, c, 0x22000); | ||
| 361 | buffer.UnmarkRegionAsCpuModified(c, 0x22000); | ||
| 362 | REQUIRE(rasterizer.Count() == 0x22); | ||
| 363 | buffer.MarkRegionAsCpuModified(c + PAGE * 0x1B, PAGE); | ||
| 364 | buffer.MarkRegionAsCpuModified(c + PAGE * 0x21, PAGE); | ||
| 365 | buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable { | ||
| 366 | static constexpr std::array<u64, 2> offsets{PAGE * 0x1B, PAGE * 0x21}; | ||
| 367 | static constexpr std::array<u64, 2> sizes{PAGE, PAGE}; | ||
| 368 | REQUIRE(offset == offsets.at(i)); | ||
| 369 | REQUIRE(size == sizes.at(i)); | ||
| 370 | ++i; | ||
| 371 | }); | ||
| 372 | } | ||
| 373 | |||
| 374 | TEST_CASE("BufferBase: Single page modified range", "[video_core]") { | ||
| 375 | RasterizerInterface rasterizer; | ||
| 376 | BufferBase buffer(rasterizer, c, PAGE); | ||
| 377 | REQUIRE(buffer.IsRegionCpuModified(c, PAGE)); | ||
| 378 | buffer.UnmarkRegionAsCpuModified(c, PAGE); | ||
| 379 | REQUIRE(!buffer.IsRegionCpuModified(c, PAGE)); | ||
| 380 | } | ||
| 381 | |||
| 382 | TEST_CASE("BufferBase: Two page modified range", "[video_core]") { | ||
| 383 | RasterizerInterface rasterizer; | ||
| 384 | BufferBase buffer(rasterizer, c, PAGE * 2); | ||
| 385 | REQUIRE(buffer.IsRegionCpuModified(c, PAGE)); | ||
| 386 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE)); | ||
| 387 | REQUIRE(buffer.IsRegionCpuModified(c, PAGE * 2)); | ||
| 388 | buffer.UnmarkRegionAsCpuModified(c, PAGE); | ||
| 389 | REQUIRE(!buffer.IsRegionCpuModified(c, PAGE)); | ||
| 390 | } | ||
| 391 | |||
| 392 | TEST_CASE("BufferBase: Multi word modified ranges", "[video_core]") { | ||
| 393 | for (int offset = 0; offset < 4; ++offset) { | ||
| 394 | const VAddr address = c + WORD * offset; | ||
| 395 | RasterizerInterface rasterizer; | ||
| 396 | BufferBase buffer(rasterizer, address, WORD * 4); | ||
| 397 | REQUIRE(buffer.IsRegionCpuModified(address, PAGE)); | ||
| 398 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 48, PAGE)); | ||
| 399 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 56, PAGE)); | ||
| 400 | |||
| 401 | buffer.UnmarkRegionAsCpuModified(address + PAGE * 32, PAGE); | ||
| 402 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE, WORD)); | ||
| 403 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE)); | ||
| 404 | REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE)); | ||
| 405 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 33, PAGE)); | ||
| 406 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE * 2)); | ||
| 407 | REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2)); | ||
| 408 | |||
| 409 | buffer.UnmarkRegionAsCpuModified(address + PAGE * 33, PAGE); | ||
| 410 | REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2)); | ||
| 411 | } | ||
| 412 | } | ||
| 413 | |||
| 414 | TEST_CASE("BufferBase: Single page in large buffer", "[video_core]") { | ||
| 415 | RasterizerInterface rasterizer; | ||
| 416 | BufferBase buffer(rasterizer, c, WORD * 16); | ||
| 417 | buffer.UnmarkRegionAsCpuModified(c, WORD * 16); | ||
| 418 | REQUIRE(!buffer.IsRegionCpuModified(c, WORD * 16)); | ||
| 419 | |||
| 420 | buffer.MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE); | ||
| 421 | REQUIRE(buffer.IsRegionCpuModified(c, WORD * 16)); | ||
| 422 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 10, WORD * 2)); | ||
| 423 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 11, WORD * 2)); | ||
| 424 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12, WORD * 2)); | ||
| 425 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8)); | ||
| 426 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE * 8)); | ||
| 427 | REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE)); | ||
| 428 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 7, PAGE * 2)); | ||
| 429 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 8, PAGE * 2)); | ||
| 430 | } | ||
| 431 | |||
| 432 | TEST_CASE("BufferBase: Out of bounds region query") { | ||
| 433 | RasterizerInterface rasterizer; | ||
| 434 | BufferBase buffer(rasterizer, c, WORD * 16); | ||
| 435 | REQUIRE(!buffer.IsRegionCpuModified(c - PAGE, PAGE)); | ||
| 436 | REQUIRE(!buffer.IsRegionCpuModified(c - PAGE * 2, PAGE)); | ||
| 437 | REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, PAGE)); | ||
| 438 | REQUIRE(buffer.IsRegionCpuModified(c + WORD * 16 - PAGE, WORD * 64)); | ||
| 439 | REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, WORD * 64)); | ||
| 440 | } | ||
| 441 | |||
| 442 | TEST_CASE("BufferBase: Wrap word regions") { | ||
| 443 | RasterizerInterface rasterizer; | ||
| 444 | BufferBase buffer(rasterizer, c, WORD * 2); | ||
| 445 | buffer.UnmarkRegionAsCpuModified(c, WORD * 2); | ||
| 446 | buffer.MarkRegionAsCpuModified(c + PAGE * 63, PAGE * 2); | ||
| 447 | REQUIRE(buffer.IsRegionCpuModified(c, WORD * 2)); | ||
| 448 | REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 62, PAGE)); | ||
| 449 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE)); | ||
| 450 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 64, PAGE)); | ||
| 451 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 2)); | ||
| 452 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 8)); | ||
| 453 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 60, PAGE * 8)); | ||
| 454 | |||
| 455 | REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16)); | ||
| 456 | buffer.MarkRegionAsCpuModified(c + PAGE * 127, PAGE); | ||
| 457 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16)); | ||
| 458 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, PAGE)); | ||
| 459 | REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 126, PAGE)); | ||
| 460 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 126, PAGE * 2)); | ||
| 461 | REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 128, WORD * 16)); | ||
| 462 | } | ||
| 463 | |||
| 464 | TEST_CASE("BufferBase: Unaligned page region query") { | ||
| 465 | RasterizerInterface rasterizer; | ||
| 466 | BufferBase buffer(rasterizer, c, WORD); | ||
| 467 | buffer.UnmarkRegionAsCpuModified(c, WORD); | ||
| 468 | buffer.MarkRegionAsCpuModified(c + 4000, 1000); | ||
| 469 | REQUIRE(buffer.IsRegionCpuModified(c, PAGE)); | ||
| 470 | REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE)); | ||
| 471 | REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1000)); | ||
| 472 | REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1)); | ||
| 473 | } | ||
diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 7a20d3a79..e01ea55ab 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt | |||
| @@ -1,6 +1,7 @@ | |||
| 1 | add_subdirectory(host_shaders) | 1 | add_subdirectory(host_shaders) |
| 2 | 2 | ||
| 3 | add_library(video_core STATIC | 3 | add_library(video_core STATIC |
| 4 | buffer_cache/buffer_base.h | ||
| 4 | buffer_cache/buffer_block.h | 5 | buffer_cache/buffer_block.h |
| 5 | buffer_cache/buffer_cache.h | 6 | buffer_cache/buffer_cache.h |
| 6 | buffer_cache/map_interval.cpp | 7 | buffer_cache/map_interval.cpp |
diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h new file mode 100644 index 000000000..fd740c2c1 --- /dev/null +++ b/src/video_core/buffer_cache/buffer_base.h | |||
| @@ -0,0 +1,495 @@ | |||
| 1 | // Copyright 2020 yuzu Emulator Project | ||
| 2 | // Licensed under GPLv2 or any later version | ||
| 3 | // Refer to the license.txt file included. | ||
| 4 | |||
| 5 | #pragma once | ||
| 6 | |||
| 7 | #include <algorithm> | ||
| 8 | #include <bit> | ||
| 9 | #include <limits> | ||
| 10 | #include <utility> | ||
| 11 | |||
| 12 | #include "common/alignment.h" | ||
| 13 | #include "common/common_funcs.h" | ||
| 14 | #include "common/common_types.h" | ||
| 15 | #include "common/div_ceil.h" | ||
| 16 | #include "core/memory.h" | ||
| 17 | |||
| 18 | namespace VideoCommon { | ||
| 19 | |||
| 20 | enum class BufferFlagBits { | ||
| 21 | Picked = 1 << 0, | ||
| 22 | }; | ||
| 23 | DECLARE_ENUM_FLAG_OPERATORS(BufferFlagBits) | ||
| 24 | |||
| 25 | /// Tag for creating null buffers with no storage or size | ||
| 26 | struct NullBufferParams {}; | ||
| 27 | |||
| 28 | /** | ||
| 29 | * Range tracking buffer container. | ||
| 30 | * | ||
| 31 | * It keeps track of the modified CPU and GPU ranges on a CPU page granularity, notifying the given | ||
| 32 | * rasterizer about state changes in the tracking behavior of the buffer. | ||
| 33 | * | ||
| 34 | * The buffer size and address is forcefully aligned to CPU page boundaries. | ||
| 35 | */ | ||
| 36 | template <class RasterizerInterface> | ||
| 37 | class BufferBase { | ||
| 38 | static constexpr u64 PAGES_PER_WORD = 64; | ||
| 39 | static constexpr u64 BYTES_PER_PAGE = Core::Memory::PAGE_SIZE; | ||
| 40 | static constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE; | ||
| 41 | |||
| 42 | /// Vector tracking modified pages tightly packed with small vector optimization | ||
| 43 | union WrittenWords { | ||
| 44 | /// Returns the pointer to the words state | ||
| 45 | [[nodiscard]] const u64* Pointer(bool is_short) const noexcept { | ||
| 46 | return is_short ? &stack : heap; | ||
| 47 | } | ||
| 48 | |||
| 49 | /// Returns the pointer to the words state | ||
| 50 | [[nodiscard]] u64* Pointer(bool is_short) noexcept { | ||
| 51 | return is_short ? &stack : heap; | ||
| 52 | } | ||
| 53 | |||
| 54 | u64 stack = 0; ///< Small buffers storage | ||
| 55 | u64* heap; ///< Not-small buffers pointer to the storage | ||
| 56 | }; | ||
| 57 | |||
| 58 | struct GpuCpuWords { | ||
| 59 | explicit GpuCpuWords() = default; | ||
| 60 | explicit GpuCpuWords(u64 size_bytes_) : size_bytes{size_bytes_} { | ||
| 61 | if (IsShort()) { | ||
| 62 | cpu.stack = ~u64{0}; | ||
| 63 | gpu.stack = 0; | ||
| 64 | } else { | ||
| 65 | // Share allocation between CPU and GPU pages and set their default values | ||
| 66 | const size_t num_words = NumWords(); | ||
| 67 | u64* const alloc = new u64[num_words * 2]; | ||
| 68 | cpu.heap = alloc; | ||
| 69 | gpu.heap = alloc + num_words; | ||
| 70 | std::fill_n(cpu.heap, num_words, ~u64{0}); | ||
| 71 | std::fill_n(gpu.heap, num_words, 0); | ||
| 72 | } | ||
| 73 | // Clean up tailing bits | ||
| 74 | const u64 last_local_page = | ||
| 75 | Common::DivCeil(size_bytes % BYTES_PER_WORD, BYTES_PER_PAGE); | ||
| 76 | const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD; | ||
| 77 | u64& last_word = cpu.Pointer(IsShort())[NumWords() - 1]; | ||
| 78 | last_word = (last_word << shift) >> shift; | ||
| 79 | } | ||
| 80 | |||
| 81 | ~GpuCpuWords() { | ||
| 82 | Release(); | ||
| 83 | } | ||
| 84 | |||
| 85 | GpuCpuWords& operator=(GpuCpuWords&& rhs) noexcept { | ||
| 86 | Release(); | ||
| 87 | size_bytes = rhs.size_bytes; | ||
| 88 | cpu = rhs.cpu; | ||
| 89 | gpu = rhs.gpu; | ||
| 90 | rhs.cpu.heap = nullptr; | ||
| 91 | return *this; | ||
| 92 | } | ||
| 93 | |||
| 94 | GpuCpuWords(GpuCpuWords&& rhs) noexcept | ||
| 95 | : size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu} { | ||
| 96 | rhs.cpu.heap = nullptr; | ||
| 97 | } | ||
| 98 | |||
| 99 | GpuCpuWords& operator=(const GpuCpuWords&) = delete; | ||
| 100 | GpuCpuWords(const GpuCpuWords&) = delete; | ||
| 101 | |||
| 102 | /// Returns true when the buffer fits in the small vector optimization | ||
| 103 | [[nodiscard]] bool IsShort() const noexcept { | ||
| 104 | return size_bytes <= BYTES_PER_WORD; | ||
| 105 | } | ||
| 106 | |||
| 107 | /// Returns the number of words of the buffer | ||
| 108 | [[nodiscard]] size_t NumWords() const noexcept { | ||
| 109 | return Common::DivCeil(size_bytes, BYTES_PER_WORD); | ||
| 110 | } | ||
| 111 | |||
| 112 | /// Release buffer resources | ||
| 113 | void Release() { | ||
| 114 | if (!IsShort()) { | ||
| 115 | // CPU written words is the base for the heap allocation | ||
| 116 | delete[] cpu.heap; | ||
| 117 | } | ||
| 118 | } | ||
| 119 | |||
| 120 | u64 size_bytes = 0; | ||
| 121 | WrittenWords cpu; | ||
| 122 | WrittenWords gpu; | ||
| 123 | }; | ||
| 124 | |||
| 125 | public: | ||
| 126 | explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes) | ||
| 127 | : rasterizer{&rasterizer_}, cpu_addr{Common::AlignDown(cpu_addr_, BYTES_PER_PAGE)}, | ||
| 128 | words(Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BYTES_PER_PAGE)) {} | ||
| 129 | |||
| 130 | explicit BufferBase(NullBufferParams) {} | ||
| 131 | |||
| 132 | BufferBase& operator=(const BufferBase&) = delete; | ||
| 133 | BufferBase(const BufferBase&) = delete; | ||
| 134 | |||
| 135 | /// Returns the inclusive CPU modified range in a begin end pair | ||
| 136 | [[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr, | ||
| 137 | u64 query_size) const noexcept { | ||
| 138 | const u64 offset = query_cpu_addr - cpu_addr; | ||
| 139 | return ModifiedRegion<false>(offset, query_size); | ||
| 140 | } | ||
| 141 | |||
| 142 | /// Returns the inclusive GPU modified range in a begin end pair | ||
| 143 | [[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr, | ||
| 144 | u64 query_size) const noexcept { | ||
| 145 | const u64 offset = query_cpu_addr - cpu_addr; | ||
| 146 | return ModifiedRegion<true>(offset, query_size); | ||
| 147 | } | ||
| 148 | |||
| 149 | /// Returns true if a region has been modified from the CPU | ||
| 150 | [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { | ||
| 151 | const u64 offset = query_cpu_addr - cpu_addr; | ||
| 152 | return IsRegionModified<false>(offset, query_size); | ||
| 153 | } | ||
| 154 | |||
| 155 | /// Returns true if a region has been modified from the GPU | ||
| 156 | [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { | ||
| 157 | const u64 offset = query_cpu_addr - cpu_addr; | ||
| 158 | return IsRegionModified<true>(offset, query_size); | ||
| 159 | } | ||
| 160 | |||
| 161 | /// Mark region as CPU modified, notifying the rasterizer about this change | ||
| 162 | void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { | ||
| 163 | ChangeRegionState<true, true>(words.cpu, dirty_cpu_addr, size); | ||
| 164 | } | ||
| 165 | |||
| 166 | /// Unmark region as CPU modified, notifying the rasterizer about this change | ||
| 167 | void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { | ||
| 168 | ChangeRegionState<false, true>(words.cpu, dirty_cpu_addr, size); | ||
| 169 | } | ||
| 170 | |||
| 171 | /// Mark region as modified from the host GPU | ||
| 172 | void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { | ||
| 173 | ChangeRegionState<true, false>(words.gpu, dirty_cpu_addr, size); | ||
| 174 | } | ||
| 175 | |||
| 176 | /// Unmark region as modified from the host GPU | ||
| 177 | void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { | ||
| 178 | ChangeRegionState<false, false>(words.gpu, dirty_cpu_addr, size); | ||
| 179 | } | ||
| 180 | |||
| 181 | /// Call 'func' for each CPU modified range and unmark those pages as CPU modified | ||
| 182 | template <typename Func> | ||
| 183 | void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) { | ||
| 184 | ForEachModifiedRange<false, true>(query_cpu_range, size, func); | ||
| 185 | } | ||
| 186 | |||
| 187 | /// Call 'func' for each GPU modified range and unmark those pages as GPU modified | ||
| 188 | template <typename Func> | ||
| 189 | void ForEachDownloadRange(VAddr query_cpu_range, u64 size, Func&& func) { | ||
| 190 | ForEachModifiedRange<true, false>(query_cpu_range, size, func); | ||
| 191 | } | ||
| 192 | |||
| 193 | /// Call 'func' for each GPU modified range and unmark those pages as GPU modified | ||
| 194 | template <typename Func> | ||
| 195 | void ForEachDownloadRange(Func&& func) { | ||
| 196 | ForEachModifiedRange<true, false>(cpu_addr, SizeBytes(), func); | ||
| 197 | } | ||
| 198 | |||
| 199 | /// Mark buffer as picked | ||
| 200 | void Pick() noexcept { | ||
| 201 | flags |= BufferFlagBits::Picked; | ||
| 202 | } | ||
| 203 | |||
| 204 | /// Unmark buffer as picked | ||
| 205 | void Unpick() noexcept { | ||
| 206 | flags &= ~BufferFlagBits::Picked; | ||
| 207 | } | ||
| 208 | |||
| 209 | /// Returns true when vaddr -> vaddr+size is fully contained in the buffer | ||
| 210 | [[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept { | ||
| 211 | return addr >= cpu_addr && addr + size <= cpu_addr + SizeBytes(); | ||
| 212 | } | ||
| 213 | |||
| 214 | /// Returns true if the buffer has been marked as picked | ||
| 215 | [[nodiscard]] bool IsPicked() const noexcept { | ||
| 216 | return True(flags & BufferFlagBits::Picked); | ||
| 217 | } | ||
| 218 | |||
| 219 | /// Returns the base CPU address of the buffer | ||
| 220 | [[nodiscard]] VAddr CpuAddr() const noexcept { | ||
| 221 | return cpu_addr; | ||
| 222 | } | ||
| 223 | |||
| 224 | /// Returns the offset relative to the given CPU address | ||
| 225 | /// @pre IsInBounds returns true | ||
| 226 | [[nodiscard]] u32 Offset(VAddr other_cpu_addr) const noexcept { | ||
| 227 | return static_cast<u32>(other_cpu_addr - cpu_addr); | ||
| 228 | } | ||
| 229 | |||
| 230 | /// Returns the size in bytes of the buffer | ||
| 231 | [[nodiscard]] u64 SizeBytes() const noexcept { | ||
| 232 | return words.size_bytes; | ||
| 233 | } | ||
| 234 | |||
| 235 | private: | ||
| 236 | /** | ||
| 237 | * Change the state of a range of pages | ||
| 238 | * | ||
| 239 | * @param written_words Pages to be marked or unmarked as modified | ||
| 240 | * @param dirty_addr Base address to mark or unmark as modified | ||
| 241 | * @param size Size in bytes to mark or unmark as modified | ||
| 242 | * | ||
| 243 | * @tparam enable True when the bits will be set to one, false for zero | ||
| 244 | * @tparam notify_rasterizer True when the rasterizer has to be notified about the changes | ||
| 245 | */ | ||
| 246 | template <bool enable, bool notify_rasterizer> | ||
| 247 | void ChangeRegionState(WrittenWords& written_words, u64 dirty_addr, | ||
| 248 | s64 size) noexcept(!notify_rasterizer) { | ||
| 249 | const s64 difference = dirty_addr - cpu_addr; | ||
| 250 | const u64 offset = std::max<s64>(difference, 0); | ||
| 251 | size += std::min<s64>(difference, 0); | ||
| 252 | if (offset >= SizeBytes() || size < 0) { | ||
| 253 | return; | ||
| 254 | } | ||
| 255 | u64* const state_words = written_words.Pointer(IsShort()); | ||
| 256 | const u64 offset_end = std::min(offset + size, SizeBytes()); | ||
| 257 | const u64 begin_page_index = offset / BYTES_PER_PAGE; | ||
| 258 | const u64 begin_word_index = begin_page_index / PAGES_PER_WORD; | ||
| 259 | const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE); | ||
| 260 | const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD); | ||
| 261 | u64 page_index = begin_page_index % PAGES_PER_WORD; | ||
| 262 | u64 word_index = begin_word_index; | ||
| 263 | while (word_index < end_word_index) { | ||
| 264 | const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD; | ||
| 265 | const u64 left_offset = | ||
| 266 | std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD; | ||
| 267 | const u64 right_offset = page_index; | ||
| 268 | u64 bits = ~u64{0}; | ||
| 269 | bits = (bits >> right_offset) << right_offset; | ||
| 270 | bits = (bits << left_offset) >> left_offset; | ||
| 271 | if constexpr (notify_rasterizer) { | ||
| 272 | NotifyRasterizer<!enable>(word_index, state_words[word_index], bits); | ||
| 273 | } | ||
| 274 | if constexpr (enable) { | ||
| 275 | state_words[word_index] |= bits; | ||
| 276 | } else { | ||
| 277 | state_words[word_index] &= ~bits; | ||
| 278 | } | ||
| 279 | page_index = 0; | ||
| 280 | ++word_index; | ||
| 281 | } | ||
| 282 | } | ||
| 283 | |||
| 284 | /** | ||
| 285 | * Notify rasterizer about changes in the CPU tracking state of a word in the buffer | ||
| 286 | * | ||
| 287 | * @param word_index Index to the word to notify to the rasterizer | ||
| 288 | * @param current_bits Current state of the word | ||
| 289 | * @param new_bits New state of the word | ||
| 290 | * | ||
| 291 | * @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages | ||
| 292 | */ | ||
| 293 | template <bool add_to_rasterizer> | ||
| 294 | void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) { | ||
| 295 | u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits; | ||
| 296 | VAddr addr = cpu_addr + word_index * BYTES_PER_WORD; | ||
| 297 | while (changed_bits != 0) { | ||
| 298 | const int empty_bits = std::countr_zero(changed_bits); | ||
| 299 | addr += empty_bits * BYTES_PER_PAGE; | ||
| 300 | changed_bits >>= empty_bits; | ||
| 301 | |||
| 302 | const u32 continuous_bits = std::countr_one(changed_bits); | ||
| 303 | const u64 size = continuous_bits * BYTES_PER_PAGE; | ||
| 304 | const VAddr begin_addr = addr; | ||
| 305 | addr += size; | ||
| 306 | changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0; | ||
| 307 | rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1); | ||
| 308 | } | ||
| 309 | } | ||
| 310 | |||
| 311 | /** | ||
| 312 | * Loop over each page in the given range, turn off those bits and notify the rasterizer if | ||
| 313 | * needed. Call the given function on each turned off range. | ||
| 314 | * | ||
| 315 | * @param query_cpu_range Base CPU address to loop over | ||
| 316 | * @param size Size in bytes of the CPU range to loop over | ||
| 317 | * @param func Function to call for each turned off region | ||
| 318 | * | ||
| 319 | * @tparam gpu True for host GPU pages, false for CPU pages | ||
| 320 | * @tparam notify_rasterizer True when the rasterizer should be notified about state changes | ||
| 321 | */ | ||
| 322 | template <bool gpu, bool notify_rasterizer, typename Func> | ||
| 323 | void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) { | ||
| 324 | const s64 difference = query_cpu_range - cpu_addr; | ||
| 325 | const u64 query_begin = std::max<s64>(difference, 0); | ||
| 326 | size += std::min<s64>(difference, 0); | ||
| 327 | if (query_begin >= SizeBytes() || size < 0) { | ||
| 328 | return; | ||
| 329 | } | ||
| 330 | const u64* const cpu_words = words.cpu.Pointer(IsShort()); | ||
| 331 | const u64 query_end = query_begin + std::min(static_cast<u64>(size), SizeBytes()); | ||
| 332 | u64* const state_words = (gpu ? words.gpu : words.cpu).Pointer(IsShort()); | ||
| 333 | u64* const words_begin = state_words + query_begin / BYTES_PER_WORD; | ||
| 334 | u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD); | ||
| 335 | |||
| 336 | const auto modified = [](u64 word) { return word != 0; }; | ||
| 337 | const auto first_modified_word = std::find_if(words_begin, words_end, modified); | ||
| 338 | if (first_modified_word == words_end) { | ||
| 339 | // Exit early when the buffer is not modified | ||
| 340 | return; | ||
| 341 | } | ||
| 342 | const auto last_modified_word = std::find_if_not(first_modified_word, words_end, modified); | ||
| 343 | |||
| 344 | const u64 word_index_begin = std::distance(state_words, first_modified_word); | ||
| 345 | const u64 word_index_end = std::distance(state_words, last_modified_word); | ||
| 346 | |||
| 347 | const unsigned local_page_begin = std::countr_zero(*first_modified_word); | ||
| 348 | const unsigned local_page_end = PAGES_PER_WORD - std::countl_zero(last_modified_word[-1]); | ||
| 349 | const u64 word_page_begin = word_index_begin * PAGES_PER_WORD; | ||
| 350 | const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD; | ||
| 351 | const u64 query_page_begin = query_begin / BYTES_PER_PAGE; | ||
| 352 | const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE); | ||
| 353 | const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin); | ||
| 354 | const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end); | ||
| 355 | const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD; | ||
| 356 | const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1; | ||
| 357 | |||
| 358 | u64 page_begin = first_word_page_begin; | ||
| 359 | u64 current_base = 0; | ||
| 360 | u64 current_size = 0; | ||
| 361 | bool on_going = false; | ||
| 362 | for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) { | ||
| 363 | const bool is_last_word = word_index + 1 == word_index_end; | ||
| 364 | const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD; | ||
| 365 | const u64 right_offset = page_begin; | ||
| 366 | const u64 left_offset = PAGES_PER_WORD - page_end; | ||
| 367 | u64 bits = ~u64{0}; | ||
| 368 | bits = (bits >> right_offset) << right_offset; | ||
| 369 | bits = (bits << left_offset) >> left_offset; | ||
| 370 | |||
| 371 | const u64 current_word = state_words[word_index] & bits; | ||
| 372 | state_words[word_index] &= ~bits; | ||
| 373 | |||
| 374 | // Exclude CPU modified pages when visiting GPU pages | ||
| 375 | const u64 word = current_word & ~(gpu ? cpu_words[word_index] : 0); | ||
| 376 | if constexpr (notify_rasterizer) { | ||
| 377 | NotifyRasterizer<true>(word_index, word, ~u64{0}); | ||
| 378 | } | ||
| 379 | u64 page = page_begin; | ||
| 380 | page_begin = 0; | ||
| 381 | |||
| 382 | while (page < page_end) { | ||
| 383 | const int empty_bits = std::countr_zero(word >> page); | ||
| 384 | if (on_going && empty_bits != 0) { | ||
| 385 | InvokeModifiedRange(func, current_size, current_base); | ||
| 386 | current_size = 0; | ||
| 387 | on_going = false; | ||
| 388 | } | ||
| 389 | page += empty_bits; | ||
| 390 | |||
| 391 | const int continuous_bits = std::countr_one(word >> page); | ||
| 392 | if (!on_going && continuous_bits != 0) { | ||
| 393 | current_base = word_index * PAGES_PER_WORD + page; | ||
| 394 | on_going = true; | ||
| 395 | } | ||
| 396 | current_size += continuous_bits; | ||
| 397 | page += continuous_bits; | ||
| 398 | } | ||
| 399 | } | ||
| 400 | if (on_going && current_size > 0) { | ||
| 401 | InvokeModifiedRange(func, current_size, current_base); | ||
| 402 | } | ||
| 403 | } | ||
| 404 | |||
| 405 | template <typename Func> | ||
| 406 | void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) { | ||
| 407 | const u64 current_size_bytes = current_size * BYTES_PER_PAGE; | ||
| 408 | const u64 offset_begin = current_base * BYTES_PER_PAGE; | ||
| 409 | const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes()); | ||
| 410 | func(offset_begin, offset_end - offset_begin); | ||
| 411 | } | ||
| 412 | |||
| 413 | /** | ||
| 414 | * Returns true when a region has been modified | ||
| 415 | * | ||
| 416 | * @param offset Offset in bytes from the start of the buffer | ||
| 417 | * @param size Size in bytes of the region to query for modifications | ||
| 418 | */ | ||
| 419 | template <bool gpu> | ||
| 420 | [[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept { | ||
| 421 | const u64* const cpu_words = words.cpu.Pointer(IsShort()); | ||
| 422 | const u64* const state_words = (gpu ? words.gpu : words.cpu).Pointer(IsShort()); | ||
| 423 | const u64 num_query_words = size / BYTES_PER_WORD + 1; | ||
| 424 | const u64 word_begin = offset / BYTES_PER_WORD; | ||
| 425 | const u64 word_end = std::min(word_begin + num_query_words, NumWords()); | ||
| 426 | const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE); | ||
| 427 | u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD; | ||
| 428 | for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) { | ||
| 429 | const u64 word = state_words[word_index] & ~(gpu ? cpu_words[word_index] : 0); | ||
| 430 | if (word == 0) { | ||
| 431 | continue; | ||
| 432 | } | ||
| 433 | const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit); | ||
| 434 | const u64 local_page_end = page_end % PAGES_PER_WORD; | ||
| 435 | const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD; | ||
| 436 | if (((word >> page_index) << page_index) << page_end_shift != 0) { | ||
| 437 | return true; | ||
| 438 | } | ||
| 439 | } | ||
| 440 | return false; | ||
| 441 | } | ||
| 442 | |||
| 443 | /** | ||
| 444 | * Returns a begin end pair with the inclusive modified region | ||
| 445 | * | ||
| 446 | * @param offset Offset in bytes from the start of the buffer | ||
| 447 | * @param size Size in bytes of the region to query for modifications | ||
| 448 | * | ||
| 449 | * @tparam True to query GPU modified pages, false for CPU pages | ||
| 450 | */ | ||
| 451 | template <bool gpu> | ||
| 452 | [[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept { | ||
| 453 | const u64* const cpu_words = words.cpu.Pointer(IsShort()); | ||
| 454 | const u64* const state_words = (gpu ? words.gpu : words.cpu).Pointer(IsShort()); | ||
| 455 | const u64 num_query_words = size / BYTES_PER_WORD + 1; | ||
| 456 | const u64 word_begin = offset / BYTES_PER_WORD; | ||
| 457 | const u64 word_end = std::min(word_begin + num_query_words, NumWords()); | ||
| 458 | const u64 page_base = offset / BYTES_PER_PAGE; | ||
| 459 | const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE); | ||
| 460 | u64 begin = std::numeric_limits<u64>::max(); | ||
| 461 | u64 end = 0; | ||
| 462 | for (u64 word_index = word_begin; word_index < word_end; ++word_index) { | ||
| 463 | const u64 word = state_words[word_index] & ~(gpu ? cpu_words[word_index] : 0); | ||
| 464 | if (word == 0) { | ||
| 465 | continue; | ||
| 466 | } | ||
| 467 | const u64 local_page_begin = std::countr_zero(word); | ||
| 468 | const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word); | ||
| 469 | const u64 page_index = word_index * PAGES_PER_WORD; | ||
| 470 | const u64 page_begin = std::max(page_index + local_page_begin, page_base); | ||
| 471 | const u64 page_end = std::min(page_index + local_page_end, page_limit); | ||
| 472 | begin = std::min(begin, page_begin); | ||
| 473 | end = std::max(end, page_end); | ||
| 474 | } | ||
| 475 | static constexpr std::pair<u64, u64> EMPTY{0, 0}; | ||
| 476 | return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY; | ||
| 477 | } | ||
| 478 | |||
| 479 | /// Returns the number of words of the buffer | ||
| 480 | [[nodiscard]] size_t NumWords() const noexcept { | ||
| 481 | return words.NumWords(); | ||
| 482 | } | ||
| 483 | |||
| 484 | /// Returns true when the buffer fits in the small vector optimization | ||
| 485 | [[nodiscard]] bool IsShort() const noexcept { | ||
| 486 | return words.IsShort(); | ||
| 487 | } | ||
| 488 | |||
| 489 | RasterizerInterface* rasterizer = nullptr; | ||
| 490 | VAddr cpu_addr = 0; | ||
| 491 | GpuCpuWords words; | ||
| 492 | BufferFlagBits flags{}; | ||
| 493 | }; | ||
| 494 | |||
| 495 | } // namespace VideoCommon | ||