summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar ReinUsesLisp2020-12-29 21:16:57 -0300
committerGravatar ReinUsesLisp2020-12-29 21:54:49 -0300
commitb3587102d160fb74a12935a79f06ee8a12712f12 (patch)
tree2afb7123bafef085cb97fc97f96be1aaf8c3a660
parentMerge pull request #5248 from ReinUsesLisp/update-dynarmic (diff)
downloadyuzu-b3587102d160fb74a12935a79f06ee8a12712f12.tar.gz
yuzu-b3587102d160fb74a12935a79f06ee8a12712f12.tar.xz
yuzu-b3587102d160fb74a12935a79f06ee8a12712f12.zip
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t pair containing 2 bits at the bottom and then the pointer. These bits are currently unused thanks to alignment requirements. Configure Dynarmic to mask out these bits on pointer reads. While we are at it, remove some unused attributes carried over from Citra. Read/Write and other hot functions use a two step unpacking process that is less readable to stop MSVC from emitting an extra AND instruction in the hot path: mov rdi,rcx shr rdx,0Ch mov r8,qword ptr [rax+8] mov rax,qword ptr [r8+rdx*8] mov rdx,rax -and al,3 and rdx,0FFFFFFFFFFFFFFFCh je Core::Memory::Memory::Impl::Read<unsigned char> mov rax,qword ptr [vaddr] movzx eax,byte ptr [rdx+rax]
Diffstat (limited to '')
-rw-r--r--src/common/page_table.cpp10
-rw-r--r--src/common/page_table.h68
-rw-r--r--src/common/virtual_buffer.h10
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_32.cpp1
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.cpp1
-rw-r--r--src/core/hle/kernel/memory/page_table.cpp2
-rw-r--r--src/core/memory.cpp187
7 files changed, 132 insertions, 147 deletions
diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp
index bccea0894..8fd8620fd 100644
--- a/src/common/page_table.cpp
+++ b/src/common/page_table.cpp
@@ -10,16 +10,10 @@ PageTable::PageTable() = default;
10 10
11PageTable::~PageTable() noexcept = default; 11PageTable::~PageTable() noexcept = default;
12 12
13void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, 13void PageTable::Resize(size_t address_space_width_in_bits, size_t page_size_in_bits) {
14 bool has_attribute) { 14 const size_t num_page_table_entries{1ULL << (address_space_width_in_bits - page_size_in_bits)};
15 const std::size_t num_page_table_entries{1ULL
16 << (address_space_width_in_bits - page_size_in_bits)};
17 pointers.resize(num_page_table_entries); 15 pointers.resize(num_page_table_entries);
18 backing_addr.resize(num_page_table_entries); 16 backing_addr.resize(num_page_table_entries);
19
20 if (has_attribute) {
21 attributes.resize(num_page_table_entries);
22 }
23} 17}
24 18
25} // namespace Common 19} // namespace Common
diff --git a/src/common/page_table.h b/src/common/page_table.h
index 9754fabf9..8d4ee9249 100644
--- a/src/common/page_table.h
+++ b/src/common/page_table.h
@@ -4,6 +4,7 @@
4 4
5#pragma once 5#pragma once
6 6
7#include <atomic>
7#include <tuple> 8#include <tuple>
8 9
9#include "common/common_types.h" 10#include "common/common_types.h"
@@ -20,10 +21,6 @@ enum class PageType : u8 {
20 /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and 21 /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
21 /// invalidation 22 /// invalidation
22 RasterizerCachedMemory, 23 RasterizerCachedMemory,
23 /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions.
24 Special,
25 /// Page is allocated for use.
26 Allocated,
27}; 24};
28 25
29struct SpecialRegion { 26struct SpecialRegion {
@@ -48,6 +45,59 @@ struct SpecialRegion {
48 * mimics the way a real CPU page table works. 45 * mimics the way a real CPU page table works.
49 */ 46 */
50struct PageTable { 47struct PageTable {
48 /// Number of bits reserved for attribute tagging.
49 /// This can be at most the guaranteed alignment of the pointers in the page table.
50 static constexpr int ATTRIBUTE_BITS = 2;
51
52 /**
53 * Pair of host pointer and page type attribute.
54 * This uses the lower bits of a given pointer to store the attribute tag.
55 * Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
56 * call. In other words, they are guaranteed to be synchronized at all times.
57 */
58 class PageInfo {
59 public:
60 /// Returns the page pointer
61 [[nodiscard]] u8* Pointer() const noexcept {
62 return ExtractPointer(raw.load(std::memory_order_relaxed));
63 }
64
65 /// Returns the page type attribute
66 [[nodiscard]] PageType Type() const noexcept {
67 return ExtractType(raw.load(std::memory_order_relaxed));
68 }
69
70 /// Returns the page pointer and attribute pair, extracted from the same atomic read
71 [[nodiscard]] std::pair<u8*, PageType> PointerType() const noexcept {
72 const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
73 return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
74 }
75
76 /// Returns the raw representation of the page information.
77 /// Use ExtractPointer and ExtractType to unpack the value.
78 [[nodiscard]] uintptr_t Raw() const noexcept {
79 return raw.load(std::memory_order_relaxed);
80 }
81
82 /// Write a page pointer and type pair atomically
83 void Store(u8* pointer, PageType type) noexcept {
84 raw.store(reinterpret_cast<uintptr_t>(pointer) | static_cast<uintptr_t>(type));
85 }
86
87 /// Unpack a pointer from a page info raw representation
88 [[nodiscard]] static u8* ExtractPointer(uintptr_t raw) noexcept {
89 return reinterpret_cast<u8*>(raw & (~uintptr_t{0} << ATTRIBUTE_BITS));
90 }
91
92 /// Unpack a page type from a page info raw representation
93 [[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
94 return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
95 }
96
97 private:
98 std::atomic<uintptr_t> raw;
99 };
100
51 PageTable(); 101 PageTable();
52 ~PageTable() noexcept; 102 ~PageTable() noexcept;
53 103
@@ -63,20 +113,16 @@ struct PageTable {
63 * 113 *
64 * @param address_space_width_in_bits The address size width in bits. 114 * @param address_space_width_in_bits The address size width in bits.
65 * @param page_size_in_bits The page size in bits. 115 * @param page_size_in_bits The page size in bits.
66 * @param has_attribute Whether or not this page has any backing attributes.
67 */ 116 */
68 void Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, 117 void Resize(size_t address_space_width_in_bits, size_t page_size_in_bits);
69 bool has_attribute);
70 118
71 /** 119 /**
72 * Vector of memory pointers backing each page. An entry can only be non-null if the 120 * Vector of memory pointers backing each page. An entry can only be non-null if the
73 * corresponding entry in the `attributes` vector is of type `Memory`. 121 * corresponding attribute element is of type `Memory`.
74 */ 122 */
75 VirtualBuffer<u8*> pointers; 123 VirtualBuffer<PageInfo> pointers;
76 124
77 VirtualBuffer<u64> backing_addr; 125 VirtualBuffer<u64> backing_addr;
78
79 VirtualBuffer<PageType> attributes;
80}; 126};
81 127
82} // namespace Common 128} // namespace Common
diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h
index 91d430036..fb1a6f81f 100644
--- a/src/common/virtual_buffer.h
+++ b/src/common/virtual_buffer.h
@@ -15,10 +15,12 @@ void FreeMemoryPages(void* base, std::size_t size) noexcept;
15template <typename T> 15template <typename T>
16class VirtualBuffer final { 16class VirtualBuffer final {
17public: 17public:
18 static_assert( 18 // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible
19 std::is_trivially_constructible_v<T>, 19 // using std::atomic_ref once libc++ has support for it
20 "T must be trivially constructible, as non-trivial constructors will not be executed " 20 // static_assert(
21 "with the current allocator"); 21 // std::is_trivially_constructible_v<T>,
22 // "T must be trivially constructible, as non-trivial constructors will not be executed "
23 // "with the current allocator");
22 24
23 constexpr VirtualBuffer() = default; 25 constexpr VirtualBuffer() = default;
24 explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} { 26 explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} {
diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp
index e9c74b1a6..8aaf11eee 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp
@@ -133,6 +133,7 @@ std::shared_ptr<Dynarmic::A32::Jit> ARM_Dynarmic_32::MakeJit(Common::PageTable&
133 config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>( 133 config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>(
134 page_table.pointers.data()); 134 page_table.pointers.data());
135 config.absolute_offset_page_table = true; 135 config.absolute_offset_page_table = true;
136 config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
136 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; 137 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
137 config.only_detect_misalignment_via_page_table_on_page_boundary = true; 138 config.only_detect_misalignment_via_page_table_on_page_boundary = true;
138 139
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
index 7a4eb88a2..d2e1dc724 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
@@ -152,6 +152,7 @@ std::shared_ptr<Dynarmic::A64::Jit> ARM_Dynarmic_64::MakeJit(Common::PageTable&
152 // Memory 152 // Memory
153 config.page_table = reinterpret_cast<void**>(page_table.pointers.data()); 153 config.page_table = reinterpret_cast<void**>(page_table.pointers.data());
154 config.page_table_address_space_bits = address_space_bits; 154 config.page_table_address_space_bits = address_space_bits;
155 config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
155 config.silently_mirror_page_table = false; 156 config.silently_mirror_page_table = false;
156 config.absolute_offset_page_table = true; 157 config.absolute_offset_page_table = true;
157 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; 158 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
diff --git a/src/core/hle/kernel/memory/page_table.cpp b/src/core/hle/kernel/memory/page_table.cpp
index f53a7be82..f3e8bc333 100644
--- a/src/core/hle/kernel/memory/page_table.cpp
+++ b/src/core/hle/kernel/memory/page_table.cpp
@@ -265,7 +265,7 @@ ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_t
265 physical_memory_usage = 0; 265 physical_memory_usage = 0;
266 memory_pool = pool; 266 memory_pool = pool;
267 267
268 page_table_impl.Resize(address_space_width, PageBits, true); 268 page_table_impl.Resize(address_space_width, PageBits);
269 269
270 return InitializeMemoryLayout(start, end); 270 return InitializeMemoryLayout(start, end);
271} 271}
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 54a848936..f209c4949 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -4,7 +4,6 @@
4 4
5#include <algorithm> 5#include <algorithm>
6#include <cstring> 6#include <cstring>
7#include <mutex>
8#include <optional> 7#include <optional>
9#include <utility> 8#include <utility>
10 9
@@ -68,21 +67,8 @@ struct Memory::Impl {
68 67
69 bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const { 68 bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const {
70 const auto& page_table = process.PageTable().PageTableImpl(); 69 const auto& page_table = process.PageTable().PageTableImpl();
71 70 const auto [pointer, type] = page_table.pointers[vaddr >> PAGE_BITS].PointerType();
72 const u8* const page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; 71 return pointer != nullptr || type == Common::PageType::RasterizerCachedMemory;
73 if (page_pointer != nullptr) {
74 return true;
75 }
76
77 if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) {
78 return true;
79 }
80
81 if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) {
82 return false;
83 }
84
85 return false;
86 } 72 }
87 73
88 bool IsValidVirtualAddress(VAddr vaddr) const { 74 bool IsValidVirtualAddress(VAddr vaddr) const {
@@ -100,17 +86,15 @@ struct Memory::Impl {
100 } 86 }
101 87
102 u8* GetPointer(const VAddr vaddr) const { 88 u8* GetPointer(const VAddr vaddr) const {
103 u8* const page_pointer{current_page_table->pointers[vaddr >> PAGE_BITS]}; 89 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
104 if (page_pointer) { 90 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
105 return page_pointer + vaddr; 91 return pointer + vaddr;
106 } 92 }
107 93 const auto type = Common::PageTable::PageInfo::ExtractType(raw_pointer);
108 if (current_page_table->attributes[vaddr >> PAGE_BITS] == 94 if (type == Common::PageType::RasterizerCachedMemory) {
109 Common::PageType::RasterizerCachedMemory) {
110 return GetPointerFromRasterizerCachedMemory(vaddr); 95 return GetPointerFromRasterizerCachedMemory(vaddr);
111 } 96 }
112 97 return nullptr;
113 return {};
114 } 98 }
115 99
116 u8 Read8(const VAddr addr) { 100 u8 Read8(const VAddr addr) {
@@ -222,7 +206,8 @@ struct Memory::Impl {
222 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 206 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
223 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 207 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
224 208
225 switch (page_table.attributes[page_index]) { 209 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
210 switch (type) {
226 case Common::PageType::Unmapped: { 211 case Common::PageType::Unmapped: {
227 LOG_ERROR(HW_Memory, 212 LOG_ERROR(HW_Memory,
228 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 213 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -231,10 +216,8 @@ struct Memory::Impl {
231 break; 216 break;
232 } 217 }
233 case Common::PageType::Memory: { 218 case Common::PageType::Memory: {
234 DEBUG_ASSERT(page_table.pointers[page_index]); 219 DEBUG_ASSERT(pointer);
235 220 const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
236 const u8* const src_ptr =
237 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
238 std::memcpy(dest_buffer, src_ptr, copy_amount); 221 std::memcpy(dest_buffer, src_ptr, copy_amount);
239 break; 222 break;
240 } 223 }
@@ -268,7 +251,8 @@ struct Memory::Impl {
268 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 251 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
269 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 252 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
270 253
271 switch (page_table.attributes[page_index]) { 254 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
255 switch (type) {
272 case Common::PageType::Unmapped: { 256 case Common::PageType::Unmapped: {
273 LOG_ERROR(HW_Memory, 257 LOG_ERROR(HW_Memory,
274 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 258 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -277,10 +261,8 @@ struct Memory::Impl {
277 break; 261 break;
278 } 262 }
279 case Common::PageType::Memory: { 263 case Common::PageType::Memory: {
280 DEBUG_ASSERT(page_table.pointers[page_index]); 264 DEBUG_ASSERT(pointer);
281 265 const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
282 const u8* const src_ptr =
283 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
284 std::memcpy(dest_buffer, src_ptr, copy_amount); 266 std::memcpy(dest_buffer, src_ptr, copy_amount);
285 break; 267 break;
286 } 268 }
@@ -320,7 +302,8 @@ struct Memory::Impl {
320 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 302 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
321 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 303 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
322 304
323 switch (page_table.attributes[page_index]) { 305 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
306 switch (type) {
324 case Common::PageType::Unmapped: { 307 case Common::PageType::Unmapped: {
325 LOG_ERROR(HW_Memory, 308 LOG_ERROR(HW_Memory,
326 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 309 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -328,10 +311,8 @@ struct Memory::Impl {
328 break; 311 break;
329 } 312 }
330 case Common::PageType::Memory: { 313 case Common::PageType::Memory: {
331 DEBUG_ASSERT(page_table.pointers[page_index]); 314 DEBUG_ASSERT(pointer);
332 315 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
333 u8* const dest_ptr =
334 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
335 std::memcpy(dest_ptr, src_buffer, copy_amount); 316 std::memcpy(dest_ptr, src_buffer, copy_amount);
336 break; 317 break;
337 } 318 }
@@ -364,7 +345,8 @@ struct Memory::Impl {
364 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 345 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
365 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 346 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
366 347
367 switch (page_table.attributes[page_index]) { 348 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
349 switch (type) {
368 case Common::PageType::Unmapped: { 350 case Common::PageType::Unmapped: {
369 LOG_ERROR(HW_Memory, 351 LOG_ERROR(HW_Memory,
370 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 352 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -372,10 +354,8 @@ struct Memory::Impl {
372 break; 354 break;
373 } 355 }
374 case Common::PageType::Memory: { 356 case Common::PageType::Memory: {
375 DEBUG_ASSERT(page_table.pointers[page_index]); 357 DEBUG_ASSERT(pointer);
376 358 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
377 u8* const dest_ptr =
378 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
379 std::memcpy(dest_ptr, src_buffer, copy_amount); 359 std::memcpy(dest_ptr, src_buffer, copy_amount);
380 break; 360 break;
381 } 361 }
@@ -414,7 +394,8 @@ struct Memory::Impl {
414 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 394 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
415 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 395 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
416 396
417 switch (page_table.attributes[page_index]) { 397 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
398 switch (type) {
418 case Common::PageType::Unmapped: { 399 case Common::PageType::Unmapped: {
419 LOG_ERROR(HW_Memory, 400 LOG_ERROR(HW_Memory,
420 "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 401 "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -422,10 +403,8 @@ struct Memory::Impl {
422 break; 403 break;
423 } 404 }
424 case Common::PageType::Memory: { 405 case Common::PageType::Memory: {
425 DEBUG_ASSERT(page_table.pointers[page_index]); 406 DEBUG_ASSERT(pointer);
426 407 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
427 u8* dest_ptr =
428 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
429 std::memset(dest_ptr, 0, copy_amount); 408 std::memset(dest_ptr, 0, copy_amount);
430 break; 409 break;
431 } 410 }
@@ -461,7 +440,8 @@ struct Memory::Impl {
461 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 440 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
462 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 441 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
463 442
464 switch (page_table.attributes[page_index]) { 443 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
444 switch (type) {
465 case Common::PageType::Unmapped: { 445 case Common::PageType::Unmapped: {
466 LOG_ERROR(HW_Memory, 446 LOG_ERROR(HW_Memory,
467 "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 447 "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -470,9 +450,8 @@ struct Memory::Impl {
470 break; 450 break;
471 } 451 }
472 case Common::PageType::Memory: { 452 case Common::PageType::Memory: {
473 DEBUG_ASSERT(page_table.pointers[page_index]); 453 DEBUG_ASSERT(pointer);
474 const u8* src_ptr = 454 const u8* src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
475 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
476 WriteBlock(process, dest_addr, src_ptr, copy_amount); 455 WriteBlock(process, dest_addr, src_ptr, copy_amount);
477 break; 456 break;
478 } 457 }
@@ -498,34 +477,19 @@ struct Memory::Impl {
498 return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size); 477 return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size);
499 } 478 }
500 479
501 struct PageEntry {
502 u8* const pointer;
503 const Common::PageType attribute;
504 };
505
506 PageEntry SafePageEntry(std::size_t base) const {
507 std::lock_guard lock{rasterizer_cache_guard};
508 return {
509 .pointer = current_page_table->pointers[base],
510 .attribute = current_page_table->attributes[base],
511 };
512 }
513
514 void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { 480 void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
515 std::lock_guard lock{rasterizer_cache_guard};
516 if (vaddr == 0) { 481 if (vaddr == 0) {
517 return; 482 return;
518 } 483 }
519
520 // Iterate over a contiguous CPU address space, which corresponds to the specified GPU 484 // Iterate over a contiguous CPU address space, which corresponds to the specified GPU
521 // address space, marking the region as un/cached. The region is marked un/cached at a 485 // address space, marking the region as un/cached. The region is marked un/cached at a
522 // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size 486 // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size
523 // is different). This assumes the specified GPU address region is contiguous as well. 487 // is different). This assumes the specified GPU address region is contiguous as well.
524 488
525 u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1; 489 const u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1;
526 for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) { 490 for (u64 i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) {
527 Common::PageType& page_type{current_page_table->attributes[vaddr >> PAGE_BITS]}; 491 const Common::PageType page_type{
528 492 current_page_table->pointers[vaddr >> PAGE_BITS].Type()};
529 if (cached) { 493 if (cached) {
530 // Switch page type to cached if now cached 494 // Switch page type to cached if now cached
531 switch (page_type) { 495 switch (page_type) {
@@ -534,8 +498,8 @@ struct Memory::Impl {
534 // space, for example, a system module need not have a VRAM mapping. 498 // space, for example, a system module need not have a VRAM mapping.
535 break; 499 break;
536 case Common::PageType::Memory: 500 case Common::PageType::Memory:
537 page_type = Common::PageType::RasterizerCachedMemory; 501 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
538 current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; 502 nullptr, Common::PageType::RasterizerCachedMemory);
539 break; 503 break;
540 case Common::PageType::RasterizerCachedMemory: 504 case Common::PageType::RasterizerCachedMemory:
541 // There can be more than one GPU region mapped per CPU region, so it's common 505 // There can be more than one GPU region mapped per CPU region, so it's common
@@ -556,16 +520,16 @@ struct Memory::Impl {
556 // that this area is already unmarked as cached. 520 // that this area is already unmarked as cached.
557 break; 521 break;
558 case Common::PageType::RasterizerCachedMemory: { 522 case Common::PageType::RasterizerCachedMemory: {
559 u8* pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)}; 523 u8* const pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)};
560 if (pointer == nullptr) { 524 if (pointer == nullptr) {
561 // It's possible that this function has been called while updating the 525 // It's possible that this function has been called while updating the
562 // pagetable after unmapping a VMA. In that case the underlying VMA will no 526 // pagetable after unmapping a VMA. In that case the underlying VMA will no
563 // longer exist, and we should just leave the pagetable entry blank. 527 // longer exist, and we should just leave the pagetable entry blank.
564 page_type = Common::PageType::Unmapped; 528 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
529 nullptr, Common::PageType::Unmapped);
565 } else { 530 } else {
566 current_page_table->pointers[vaddr >> PAGE_BITS] = 531 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
567 pointer - (vaddr & ~PAGE_MASK); 532 pointer - (vaddr & ~PAGE_MASK), Common::PageType::Memory);
568 page_type = Common::PageType::Memory;
569 } 533 }
570 break; 534 break;
571 } 535 }
@@ -595,7 +559,7 @@ struct Memory::Impl {
595 auto& gpu = system.GPU(); 559 auto& gpu = system.GPU();
596 for (u64 i = 0; i < size; i++) { 560 for (u64 i = 0; i < size; i++) {
597 const auto page = base + i; 561 const auto page = base + i;
598 if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) { 562 if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) {
599 gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE); 563 gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE);
600 } 564 }
601 } 565 }
@@ -610,20 +574,18 @@ struct Memory::Impl {
610 "Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE); 574 "Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE);
611 575
612 while (base != end) { 576 while (base != end) {
613 page_table.attributes[base] = type; 577 page_table.pointers[base].Store(nullptr, type);
614 page_table.pointers[base] = nullptr;
615 page_table.backing_addr[base] = 0; 578 page_table.backing_addr[base] = 0;
616 579
617 base += 1; 580 base += 1;
618 } 581 }
619 } else { 582 } else {
620 while (base != end) { 583 while (base != end) {
621 page_table.pointers[base] = 584 page_table.pointers[base].Store(
622 system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS); 585 system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS), type);
623 page_table.attributes[base] = type;
624 page_table.backing_addr[base] = target - (base << PAGE_BITS); 586 page_table.backing_addr[base] = target - (base << PAGE_BITS);
625 587
626 ASSERT_MSG(page_table.pointers[base], 588 ASSERT_MSG(page_table.pointers[base].Pointer(),
627 "memory mapping base yield a nullptr within the table"); 589 "memory mapping base yield a nullptr within the table");
628 590
629 base += 1; 591 base += 1;
@@ -646,21 +608,13 @@ struct Memory::Impl {
646 template <typename T> 608 template <typename T>
647 T Read(const VAddr vaddr) { 609 T Read(const VAddr vaddr) {
648 // Avoid adding any extra logic to this fast-path block 610 // Avoid adding any extra logic to this fast-path block
649 if (const u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { 611 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
612 if (const u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
650 T value; 613 T value;
651 std::memcpy(&value, &pointer[vaddr], sizeof(T)); 614 std::memcpy(&value, &pointer[vaddr], sizeof(T));
652 return value; 615 return value;
653 } 616 }
654 617 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
655 // Otherwise, we need to grab the page with a lock, in case it is currently being modified
656 const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
657 if (entry.pointer) {
658 T value;
659 std::memcpy(&value, &entry.pointer[vaddr], sizeof(T));
660 return value;
661 }
662
663 switch (entry.attribute) {
664 case Common::PageType::Unmapped: 618 case Common::PageType::Unmapped:
665 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); 619 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
666 return 0; 620 return 0;
@@ -692,20 +646,12 @@ struct Memory::Impl {
692 template <typename T> 646 template <typename T>
693 void Write(const VAddr vaddr, const T data) { 647 void Write(const VAddr vaddr, const T data) {
694 // Avoid adding any extra logic to this fast-path block 648 // Avoid adding any extra logic to this fast-path block
695 if (u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { 649 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
650 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
696 std::memcpy(&pointer[vaddr], &data, sizeof(T)); 651 std::memcpy(&pointer[vaddr], &data, sizeof(T));
697 return; 652 return;
698 } 653 }
699 654 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
700 // Otherwise, we need to grab the page with a lock, in case it is currently being modified
701 const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
702 if (entry.pointer) {
703 // Memory was mapped, we are done
704 std::memcpy(&entry.pointer[vaddr], &data, sizeof(T));
705 return;
706 }
707
708 switch (entry.attribute) {
709 case Common::PageType::Unmapped: 655 case Common::PageType::Unmapped:
710 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 656 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
711 static_cast<u32>(data), vaddr); 657 static_cast<u32>(data), vaddr);
@@ -726,15 +672,13 @@ struct Memory::Impl {
726 672
727 template <typename T> 673 template <typename T>
728 bool WriteExclusive(const VAddr vaddr, const T data, const T expected) { 674 bool WriteExclusive(const VAddr vaddr, const T data, const T expected) {
729 u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; 675 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
730 if (page_pointer != nullptr) { 676 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
731 // NOTE: Avoid adding any extra logic to this fast-path block 677 // NOTE: Avoid adding any extra logic to this fast-path block
732 auto* pointer = reinterpret_cast<volatile T*>(&page_pointer[vaddr]); 678 const auto volatile_pointer = reinterpret_cast<volatile T*>(&pointer[vaddr]);
733 return Common::AtomicCompareAndSwap(pointer, data, expected); 679 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
734 } 680 }
735 681 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
736 const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
737 switch (type) {
738 case Common::PageType::Unmapped: 682 case Common::PageType::Unmapped:
739 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 683 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
740 static_cast<u32>(data), vaddr); 684 static_cast<u32>(data), vaddr);
@@ -755,15 +699,13 @@ struct Memory::Impl {
755 } 699 }
756 700
757 bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) { 701 bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) {
758 u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; 702 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
759 if (page_pointer != nullptr) { 703 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
760 // NOTE: Avoid adding any extra logic to this fast-path block 704 // NOTE: Avoid adding any extra logic to this fast-path block
761 auto* pointer = reinterpret_cast<volatile u64*>(&page_pointer[vaddr]); 705 const auto volatile_pointer = reinterpret_cast<volatile u64*>(&pointer[vaddr]);
762 return Common::AtomicCompareAndSwap(pointer, data, expected); 706 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
763 } 707 }
764 708 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
765 const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
766 switch (type) {
767 case Common::PageType::Unmapped: 709 case Common::PageType::Unmapped:
768 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8, 710 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8,
769 static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr); 711 static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr);
@@ -783,7 +725,6 @@ struct Memory::Impl {
783 return true; 725 return true;
784 } 726 }
785 727
786 mutable std::mutex rasterizer_cache_guard;
787 Common::PageTable* current_page_table = nullptr; 728 Common::PageTable* current_page_table = nullptr;
788 Core::System& system; 729 Core::System& system;
789}; 730};