summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/common/page_table.cpp10
-rw-r--r--src/common/page_table.h68
-rw-r--r--src/common/virtual_buffer.h10
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_32.cpp1
-rw-r--r--src/core/arm/dynarmic/arm_dynarmic_64.cpp1
-rw-r--r--src/core/hle/kernel/memory/page_table.cpp2
-rw-r--r--src/core/memory.cpp187
7 files changed, 132 insertions, 147 deletions
diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp
index bccea0894..8fd8620fd 100644
--- a/src/common/page_table.cpp
+++ b/src/common/page_table.cpp
@@ -10,16 +10,10 @@ PageTable::PageTable() = default;
10 10
11PageTable::~PageTable() noexcept = default; 11PageTable::~PageTable() noexcept = default;
12 12
13void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, 13void PageTable::Resize(size_t address_space_width_in_bits, size_t page_size_in_bits) {
14 bool has_attribute) { 14 const size_t num_page_table_entries{1ULL << (address_space_width_in_bits - page_size_in_bits)};
15 const std::size_t num_page_table_entries{1ULL
16 << (address_space_width_in_bits - page_size_in_bits)};
17 pointers.resize(num_page_table_entries); 15 pointers.resize(num_page_table_entries);
18 backing_addr.resize(num_page_table_entries); 16 backing_addr.resize(num_page_table_entries);
19
20 if (has_attribute) {
21 attributes.resize(num_page_table_entries);
22 }
23} 17}
24 18
25} // namespace Common 19} // namespace Common
diff --git a/src/common/page_table.h b/src/common/page_table.h
index 9754fabf9..8d4ee9249 100644
--- a/src/common/page_table.h
+++ b/src/common/page_table.h
@@ -4,6 +4,7 @@
4 4
5#pragma once 5#pragma once
6 6
7#include <atomic>
7#include <tuple> 8#include <tuple>
8 9
9#include "common/common_types.h" 10#include "common/common_types.h"
@@ -20,10 +21,6 @@ enum class PageType : u8 {
20 /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and 21 /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
21 /// invalidation 22 /// invalidation
22 RasterizerCachedMemory, 23 RasterizerCachedMemory,
23 /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions.
24 Special,
25 /// Page is allocated for use.
26 Allocated,
27}; 24};
28 25
29struct SpecialRegion { 26struct SpecialRegion {
@@ -48,6 +45,59 @@ struct SpecialRegion {
48 * mimics the way a real CPU page table works. 45 * mimics the way a real CPU page table works.
49 */ 46 */
50struct PageTable { 47struct PageTable {
48 /// Number of bits reserved for attribute tagging.
49 /// This can be at most the guaranteed alignment of the pointers in the page table.
50 static constexpr int ATTRIBUTE_BITS = 2;
51
52 /**
53 * Pair of host pointer and page type attribute.
54 * This uses the lower bits of a given pointer to store the attribute tag.
55 * Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
56 * call. In other words, they are guaranteed to be synchronized at all times.
57 */
58 class PageInfo {
59 public:
60 /// Returns the page pointer
61 [[nodiscard]] u8* Pointer() const noexcept {
62 return ExtractPointer(raw.load(std::memory_order_relaxed));
63 }
64
65 /// Returns the page type attribute
66 [[nodiscard]] PageType Type() const noexcept {
67 return ExtractType(raw.load(std::memory_order_relaxed));
68 }
69
70 /// Returns the page pointer and attribute pair, extracted from the same atomic read
71 [[nodiscard]] std::pair<u8*, PageType> PointerType() const noexcept {
72 const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
73 return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
74 }
75
76 /// Returns the raw representation of the page information.
77 /// Use ExtractPointer and ExtractType to unpack the value.
78 [[nodiscard]] uintptr_t Raw() const noexcept {
79 return raw.load(std::memory_order_relaxed);
80 }
81
82 /// Write a page pointer and type pair atomically
83 void Store(u8* pointer, PageType type) noexcept {
84 raw.store(reinterpret_cast<uintptr_t>(pointer) | static_cast<uintptr_t>(type));
85 }
86
87 /// Unpack a pointer from a page info raw representation
88 [[nodiscard]] static u8* ExtractPointer(uintptr_t raw) noexcept {
89 return reinterpret_cast<u8*>(raw & (~uintptr_t{0} << ATTRIBUTE_BITS));
90 }
91
92 /// Unpack a page type from a page info raw representation
93 [[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
94 return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
95 }
96
97 private:
98 std::atomic<uintptr_t> raw;
99 };
100
51 PageTable(); 101 PageTable();
52 ~PageTable() noexcept; 102 ~PageTable() noexcept;
53 103
@@ -63,20 +113,16 @@ struct PageTable {
63 * 113 *
64 * @param address_space_width_in_bits The address size width in bits. 114 * @param address_space_width_in_bits The address size width in bits.
65 * @param page_size_in_bits The page size in bits. 115 * @param page_size_in_bits The page size in bits.
66 * @param has_attribute Whether or not this page has any backing attributes.
67 */ 116 */
68 void Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits, 117 void Resize(size_t address_space_width_in_bits, size_t page_size_in_bits);
69 bool has_attribute);
70 118
71 /** 119 /**
72 * Vector of memory pointers backing each page. An entry can only be non-null if the 120 * Vector of memory pointers backing each page. An entry can only be non-null if the
73 * corresponding entry in the `attributes` vector is of type `Memory`. 121 * corresponding attribute element is of type `Memory`.
74 */ 122 */
75 VirtualBuffer<u8*> pointers; 123 VirtualBuffer<PageInfo> pointers;
76 124
77 VirtualBuffer<u64> backing_addr; 125 VirtualBuffer<u64> backing_addr;
78
79 VirtualBuffer<PageType> attributes;
80}; 126};
81 127
82} // namespace Common 128} // namespace Common
diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h
index 91d430036..fb1a6f81f 100644
--- a/src/common/virtual_buffer.h
+++ b/src/common/virtual_buffer.h
@@ -15,10 +15,12 @@ void FreeMemoryPages(void* base, std::size_t size) noexcept;
15template <typename T> 15template <typename T>
16class VirtualBuffer final { 16class VirtualBuffer final {
17public: 17public:
18 static_assert( 18 // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible
19 std::is_trivially_constructible_v<T>, 19 // using std::atomic_ref once libc++ has support for it
20 "T must be trivially constructible, as non-trivial constructors will not be executed " 20 // static_assert(
21 "with the current allocator"); 21 // std::is_trivially_constructible_v<T>,
22 // "T must be trivially constructible, as non-trivial constructors will not be executed "
23 // "with the current allocator");
22 24
23 constexpr VirtualBuffer() = default; 25 constexpr VirtualBuffer() = default;
24 explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} { 26 explicit VirtualBuffer(std::size_t count) : alloc_size{count * sizeof(T)} {
diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp
index e9c74b1a6..8aaf11eee 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp
@@ -133,6 +133,7 @@ std::shared_ptr<Dynarmic::A32::Jit> ARM_Dynarmic_32::MakeJit(Common::PageTable&
133 config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>( 133 config.page_table = reinterpret_cast<std::array<std::uint8_t*, NUM_PAGE_TABLE_ENTRIES>*>(
134 page_table.pointers.data()); 134 page_table.pointers.data());
135 config.absolute_offset_page_table = true; 135 config.absolute_offset_page_table = true;
136 config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
136 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; 137 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
137 config.only_detect_misalignment_via_page_table_on_page_boundary = true; 138 config.only_detect_misalignment_via_page_table_on_page_boundary = true;
138 139
diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
index 7a4eb88a2..d2e1dc724 100644
--- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp
@@ -152,6 +152,7 @@ std::shared_ptr<Dynarmic::A64::Jit> ARM_Dynarmic_64::MakeJit(Common::PageTable&
152 // Memory 152 // Memory
153 config.page_table = reinterpret_cast<void**>(page_table.pointers.data()); 153 config.page_table = reinterpret_cast<void**>(page_table.pointers.data());
154 config.page_table_address_space_bits = address_space_bits; 154 config.page_table_address_space_bits = address_space_bits;
155 config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
155 config.silently_mirror_page_table = false; 156 config.silently_mirror_page_table = false;
156 config.absolute_offset_page_table = true; 157 config.absolute_offset_page_table = true;
157 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; 158 config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
diff --git a/src/core/hle/kernel/memory/page_table.cpp b/src/core/hle/kernel/memory/page_table.cpp
index f53a7be82..f3e8bc333 100644
--- a/src/core/hle/kernel/memory/page_table.cpp
+++ b/src/core/hle/kernel/memory/page_table.cpp
@@ -265,7 +265,7 @@ ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_t
265 physical_memory_usage = 0; 265 physical_memory_usage = 0;
266 memory_pool = pool; 266 memory_pool = pool;
267 267
268 page_table_impl.Resize(address_space_width, PageBits, true); 268 page_table_impl.Resize(address_space_width, PageBits);
269 269
270 return InitializeMemoryLayout(start, end); 270 return InitializeMemoryLayout(start, end);
271} 271}
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 54a848936..f209c4949 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -4,7 +4,6 @@
4 4
5#include <algorithm> 5#include <algorithm>
6#include <cstring> 6#include <cstring>
7#include <mutex>
8#include <optional> 7#include <optional>
9#include <utility> 8#include <utility>
10 9
@@ -68,21 +67,8 @@ struct Memory::Impl {
68 67
69 bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const { 68 bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const {
70 const auto& page_table = process.PageTable().PageTableImpl(); 69 const auto& page_table = process.PageTable().PageTableImpl();
71 70 const auto [pointer, type] = page_table.pointers[vaddr >> PAGE_BITS].PointerType();
72 const u8* const page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; 71 return pointer != nullptr || type == Common::PageType::RasterizerCachedMemory;
73 if (page_pointer != nullptr) {
74 return true;
75 }
76
77 if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) {
78 return true;
79 }
80
81 if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) {
82 return false;
83 }
84
85 return false;
86 } 72 }
87 73
88 bool IsValidVirtualAddress(VAddr vaddr) const { 74 bool IsValidVirtualAddress(VAddr vaddr) const {
@@ -100,17 +86,15 @@ struct Memory::Impl {
100 } 86 }
101 87
102 u8* GetPointer(const VAddr vaddr) const { 88 u8* GetPointer(const VAddr vaddr) const {
103 u8* const page_pointer{current_page_table->pointers[vaddr >> PAGE_BITS]}; 89 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
104 if (page_pointer) { 90 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
105 return page_pointer + vaddr; 91 return pointer + vaddr;
106 } 92 }
107 93 const auto type = Common::PageTable::PageInfo::ExtractType(raw_pointer);
108 if (current_page_table->attributes[vaddr >> PAGE_BITS] == 94 if (type == Common::PageType::RasterizerCachedMemory) {
109 Common::PageType::RasterizerCachedMemory) {
110 return GetPointerFromRasterizerCachedMemory(vaddr); 95 return GetPointerFromRasterizerCachedMemory(vaddr);
111 } 96 }
112 97 return nullptr;
113 return {};
114 } 98 }
115 99
116 u8 Read8(const VAddr addr) { 100 u8 Read8(const VAddr addr) {
@@ -222,7 +206,8 @@ struct Memory::Impl {
222 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 206 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
223 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 207 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
224 208
225 switch (page_table.attributes[page_index]) { 209 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
210 switch (type) {
226 case Common::PageType::Unmapped: { 211 case Common::PageType::Unmapped: {
227 LOG_ERROR(HW_Memory, 212 LOG_ERROR(HW_Memory,
228 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 213 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -231,10 +216,8 @@ struct Memory::Impl {
231 break; 216 break;
232 } 217 }
233 case Common::PageType::Memory: { 218 case Common::PageType::Memory: {
234 DEBUG_ASSERT(page_table.pointers[page_index]); 219 DEBUG_ASSERT(pointer);
235 220 const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
236 const u8* const src_ptr =
237 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
238 std::memcpy(dest_buffer, src_ptr, copy_amount); 221 std::memcpy(dest_buffer, src_ptr, copy_amount);
239 break; 222 break;
240 } 223 }
@@ -268,7 +251,8 @@ struct Memory::Impl {
268 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 251 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
269 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 252 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
270 253
271 switch (page_table.attributes[page_index]) { 254 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
255 switch (type) {
272 case Common::PageType::Unmapped: { 256 case Common::PageType::Unmapped: {
273 LOG_ERROR(HW_Memory, 257 LOG_ERROR(HW_Memory,
274 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 258 "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -277,10 +261,8 @@ struct Memory::Impl {
277 break; 261 break;
278 } 262 }
279 case Common::PageType::Memory: { 263 case Common::PageType::Memory: {
280 DEBUG_ASSERT(page_table.pointers[page_index]); 264 DEBUG_ASSERT(pointer);
281 265 const u8* const src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
282 const u8* const src_ptr =
283 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
284 std::memcpy(dest_buffer, src_ptr, copy_amount); 266 std::memcpy(dest_buffer, src_ptr, copy_amount);
285 break; 267 break;
286 } 268 }
@@ -320,7 +302,8 @@ struct Memory::Impl {
320 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 302 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
321 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 303 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
322 304
323 switch (page_table.attributes[page_index]) { 305 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
306 switch (type) {
324 case Common::PageType::Unmapped: { 307 case Common::PageType::Unmapped: {
325 LOG_ERROR(HW_Memory, 308 LOG_ERROR(HW_Memory,
326 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 309 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -328,10 +311,8 @@ struct Memory::Impl {
328 break; 311 break;
329 } 312 }
330 case Common::PageType::Memory: { 313 case Common::PageType::Memory: {
331 DEBUG_ASSERT(page_table.pointers[page_index]); 314 DEBUG_ASSERT(pointer);
332 315 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
333 u8* const dest_ptr =
334 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
335 std::memcpy(dest_ptr, src_buffer, copy_amount); 316 std::memcpy(dest_ptr, src_buffer, copy_amount);
336 break; 317 break;
337 } 318 }
@@ -364,7 +345,8 @@ struct Memory::Impl {
364 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 345 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
365 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 346 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
366 347
367 switch (page_table.attributes[page_index]) { 348 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
349 switch (type) {
368 case Common::PageType::Unmapped: { 350 case Common::PageType::Unmapped: {
369 LOG_ERROR(HW_Memory, 351 LOG_ERROR(HW_Memory,
370 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 352 "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -372,10 +354,8 @@ struct Memory::Impl {
372 break; 354 break;
373 } 355 }
374 case Common::PageType::Memory: { 356 case Common::PageType::Memory: {
375 DEBUG_ASSERT(page_table.pointers[page_index]); 357 DEBUG_ASSERT(pointer);
376 358 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
377 u8* const dest_ptr =
378 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
379 std::memcpy(dest_ptr, src_buffer, copy_amount); 359 std::memcpy(dest_ptr, src_buffer, copy_amount);
380 break; 360 break;
381 } 361 }
@@ -414,7 +394,8 @@ struct Memory::Impl {
414 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 394 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
415 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 395 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
416 396
417 switch (page_table.attributes[page_index]) { 397 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
398 switch (type) {
418 case Common::PageType::Unmapped: { 399 case Common::PageType::Unmapped: {
419 LOG_ERROR(HW_Memory, 400 LOG_ERROR(HW_Memory,
420 "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 401 "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -422,10 +403,8 @@ struct Memory::Impl {
422 break; 403 break;
423 } 404 }
424 case Common::PageType::Memory: { 405 case Common::PageType::Memory: {
425 DEBUG_ASSERT(page_table.pointers[page_index]); 406 DEBUG_ASSERT(pointer);
426 407 u8* const dest_ptr = pointer + page_offset + (page_index << PAGE_BITS);
427 u8* dest_ptr =
428 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
429 std::memset(dest_ptr, 0, copy_amount); 408 std::memset(dest_ptr, 0, copy_amount);
430 break; 409 break;
431 } 410 }
@@ -461,7 +440,8 @@ struct Memory::Impl {
461 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); 440 std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
462 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); 441 const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
463 442
464 switch (page_table.attributes[page_index]) { 443 const auto [pointer, type] = page_table.pointers[page_index].PointerType();
444 switch (type) {
465 case Common::PageType::Unmapped: { 445 case Common::PageType::Unmapped: {
466 LOG_ERROR(HW_Memory, 446 LOG_ERROR(HW_Memory,
467 "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", 447 "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
@@ -470,9 +450,8 @@ struct Memory::Impl {
470 break; 450 break;
471 } 451 }
472 case Common::PageType::Memory: { 452 case Common::PageType::Memory: {
473 DEBUG_ASSERT(page_table.pointers[page_index]); 453 DEBUG_ASSERT(pointer);
474 const u8* src_ptr = 454 const u8* src_ptr = pointer + page_offset + (page_index << PAGE_BITS);
475 page_table.pointers[page_index] + page_offset + (page_index << PAGE_BITS);
476 WriteBlock(process, dest_addr, src_ptr, copy_amount); 455 WriteBlock(process, dest_addr, src_ptr, copy_amount);
477 break; 456 break;
478 } 457 }
@@ -498,34 +477,19 @@ struct Memory::Impl {
498 return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size); 477 return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size);
499 } 478 }
500 479
501 struct PageEntry {
502 u8* const pointer;
503 const Common::PageType attribute;
504 };
505
506 PageEntry SafePageEntry(std::size_t base) const {
507 std::lock_guard lock{rasterizer_cache_guard};
508 return {
509 .pointer = current_page_table->pointers[base],
510 .attribute = current_page_table->attributes[base],
511 };
512 }
513
514 void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { 480 void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
515 std::lock_guard lock{rasterizer_cache_guard};
516 if (vaddr == 0) { 481 if (vaddr == 0) {
517 return; 482 return;
518 } 483 }
519
520 // Iterate over a contiguous CPU address space, which corresponds to the specified GPU 484 // Iterate over a contiguous CPU address space, which corresponds to the specified GPU
521 // address space, marking the region as un/cached. The region is marked un/cached at a 485 // address space, marking the region as un/cached. The region is marked un/cached at a
522 // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size 486 // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size
523 // is different). This assumes the specified GPU address region is contiguous as well. 487 // is different). This assumes the specified GPU address region is contiguous as well.
524 488
525 u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1; 489 const u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1;
526 for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) { 490 for (u64 i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) {
527 Common::PageType& page_type{current_page_table->attributes[vaddr >> PAGE_BITS]}; 491 const Common::PageType page_type{
528 492 current_page_table->pointers[vaddr >> PAGE_BITS].Type()};
529 if (cached) { 493 if (cached) {
530 // Switch page type to cached if now cached 494 // Switch page type to cached if now cached
531 switch (page_type) { 495 switch (page_type) {
@@ -534,8 +498,8 @@ struct Memory::Impl {
534 // space, for example, a system module need not have a VRAM mapping. 498 // space, for example, a system module need not have a VRAM mapping.
535 break; 499 break;
536 case Common::PageType::Memory: 500 case Common::PageType::Memory:
537 page_type = Common::PageType::RasterizerCachedMemory; 501 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
538 current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; 502 nullptr, Common::PageType::RasterizerCachedMemory);
539 break; 503 break;
540 case Common::PageType::RasterizerCachedMemory: 504 case Common::PageType::RasterizerCachedMemory:
541 // There can be more than one GPU region mapped per CPU region, so it's common 505 // There can be more than one GPU region mapped per CPU region, so it's common
@@ -556,16 +520,16 @@ struct Memory::Impl {
556 // that this area is already unmarked as cached. 520 // that this area is already unmarked as cached.
557 break; 521 break;
558 case Common::PageType::RasterizerCachedMemory: { 522 case Common::PageType::RasterizerCachedMemory: {
559 u8* pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)}; 523 u8* const pointer{GetPointerFromRasterizerCachedMemory(vaddr & ~PAGE_MASK)};
560 if (pointer == nullptr) { 524 if (pointer == nullptr) {
561 // It's possible that this function has been called while updating the 525 // It's possible that this function has been called while updating the
562 // pagetable after unmapping a VMA. In that case the underlying VMA will no 526 // pagetable after unmapping a VMA. In that case the underlying VMA will no
563 // longer exist, and we should just leave the pagetable entry blank. 527 // longer exist, and we should just leave the pagetable entry blank.
564 page_type = Common::PageType::Unmapped; 528 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
529 nullptr, Common::PageType::Unmapped);
565 } else { 530 } else {
566 current_page_table->pointers[vaddr >> PAGE_BITS] = 531 current_page_table->pointers[vaddr >> PAGE_BITS].Store(
567 pointer - (vaddr & ~PAGE_MASK); 532 pointer - (vaddr & ~PAGE_MASK), Common::PageType::Memory);
568 page_type = Common::PageType::Memory;
569 } 533 }
570 break; 534 break;
571 } 535 }
@@ -595,7 +559,7 @@ struct Memory::Impl {
595 auto& gpu = system.GPU(); 559 auto& gpu = system.GPU();
596 for (u64 i = 0; i < size; i++) { 560 for (u64 i = 0; i < size; i++) {
597 const auto page = base + i; 561 const auto page = base + i;
598 if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) { 562 if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) {
599 gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE); 563 gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE);
600 } 564 }
601 } 565 }
@@ -610,20 +574,18 @@ struct Memory::Impl {
610 "Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE); 574 "Mapping memory page without a pointer @ {:016x}", base * PAGE_SIZE);
611 575
612 while (base != end) { 576 while (base != end) {
613 page_table.attributes[base] = type; 577 page_table.pointers[base].Store(nullptr, type);
614 page_table.pointers[base] = nullptr;
615 page_table.backing_addr[base] = 0; 578 page_table.backing_addr[base] = 0;
616 579
617 base += 1; 580 base += 1;
618 } 581 }
619 } else { 582 } else {
620 while (base != end) { 583 while (base != end) {
621 page_table.pointers[base] = 584 page_table.pointers[base].Store(
622 system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS); 585 system.DeviceMemory().GetPointer(target) - (base << PAGE_BITS), type);
623 page_table.attributes[base] = type;
624 page_table.backing_addr[base] = target - (base << PAGE_BITS); 586 page_table.backing_addr[base] = target - (base << PAGE_BITS);
625 587
626 ASSERT_MSG(page_table.pointers[base], 588 ASSERT_MSG(page_table.pointers[base].Pointer(),
627 "memory mapping base yield a nullptr within the table"); 589 "memory mapping base yield a nullptr within the table");
628 590
629 base += 1; 591 base += 1;
@@ -646,21 +608,13 @@ struct Memory::Impl {
646 template <typename T> 608 template <typename T>
647 T Read(const VAddr vaddr) { 609 T Read(const VAddr vaddr) {
648 // Avoid adding any extra logic to this fast-path block 610 // Avoid adding any extra logic to this fast-path block
649 if (const u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { 611 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
612 if (const u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
650 T value; 613 T value;
651 std::memcpy(&value, &pointer[vaddr], sizeof(T)); 614 std::memcpy(&value, &pointer[vaddr], sizeof(T));
652 return value; 615 return value;
653 } 616 }
654 617 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
655 // Otherwise, we need to grab the page with a lock, in case it is currently being modified
656 const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
657 if (entry.pointer) {
658 T value;
659 std::memcpy(&value, &entry.pointer[vaddr], sizeof(T));
660 return value;
661 }
662
663 switch (entry.attribute) {
664 case Common::PageType::Unmapped: 618 case Common::PageType::Unmapped:
665 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); 619 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
666 return 0; 620 return 0;
@@ -692,20 +646,12 @@ struct Memory::Impl {
692 template <typename T> 646 template <typename T>
693 void Write(const VAddr vaddr, const T data) { 647 void Write(const VAddr vaddr, const T data) {
694 // Avoid adding any extra logic to this fast-path block 648 // Avoid adding any extra logic to this fast-path block
695 if (u8* const pointer = current_page_table->pointers[vaddr >> PAGE_BITS]) { 649 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
650 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
696 std::memcpy(&pointer[vaddr], &data, sizeof(T)); 651 std::memcpy(&pointer[vaddr], &data, sizeof(T));
697 return; 652 return;
698 } 653 }
699 654 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
700 // Otherwise, we need to grab the page with a lock, in case it is currently being modified
701 const auto entry = SafePageEntry(vaddr >> PAGE_BITS);
702 if (entry.pointer) {
703 // Memory was mapped, we are done
704 std::memcpy(&entry.pointer[vaddr], &data, sizeof(T));
705 return;
706 }
707
708 switch (entry.attribute) {
709 case Common::PageType::Unmapped: 655 case Common::PageType::Unmapped:
710 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 656 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
711 static_cast<u32>(data), vaddr); 657 static_cast<u32>(data), vaddr);
@@ -726,15 +672,13 @@ struct Memory::Impl {
726 672
727 template <typename T> 673 template <typename T>
728 bool WriteExclusive(const VAddr vaddr, const T data, const T expected) { 674 bool WriteExclusive(const VAddr vaddr, const T data, const T expected) {
729 u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; 675 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
730 if (page_pointer != nullptr) { 676 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
731 // NOTE: Avoid adding any extra logic to this fast-path block 677 // NOTE: Avoid adding any extra logic to this fast-path block
732 auto* pointer = reinterpret_cast<volatile T*>(&page_pointer[vaddr]); 678 const auto volatile_pointer = reinterpret_cast<volatile T*>(&pointer[vaddr]);
733 return Common::AtomicCompareAndSwap(pointer, data, expected); 679 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
734 } 680 }
735 681 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
736 const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
737 switch (type) {
738 case Common::PageType::Unmapped: 682 case Common::PageType::Unmapped:
739 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 683 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
740 static_cast<u32>(data), vaddr); 684 static_cast<u32>(data), vaddr);
@@ -755,15 +699,13 @@ struct Memory::Impl {
755 } 699 }
756 700
757 bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) { 701 bool WriteExclusive128(const VAddr vaddr, const u128 data, const u128 expected) {
758 u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; 702 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
759 if (page_pointer != nullptr) { 703 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
760 // NOTE: Avoid adding any extra logic to this fast-path block 704 // NOTE: Avoid adding any extra logic to this fast-path block
761 auto* pointer = reinterpret_cast<volatile u64*>(&page_pointer[vaddr]); 705 const auto volatile_pointer = reinterpret_cast<volatile u64*>(&pointer[vaddr]);
762 return Common::AtomicCompareAndSwap(pointer, data, expected); 706 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
763 } 707 }
764 708 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
765 const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
766 switch (type) {
767 case Common::PageType::Unmapped: 709 case Common::PageType::Unmapped:
768 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8, 710 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8,
769 static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr); 711 static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr);
@@ -783,7 +725,6 @@ struct Memory::Impl {
783 return true; 725 return true;
784 } 726 }
785 727
786 mutable std::mutex rasterizer_cache_guard;
787 Common::PageTable* current_page_table = nullptr; 728 Common::PageTable* current_page_table = nullptr;
788 Core::System& system; 729 Core::System& system;
789}; 730};