summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/core/memory.cpp244
1 files changed, 115 insertions, 129 deletions
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 0b8e36b08..778d152dd 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -5,6 +5,10 @@
5#include <algorithm> 5#include <algorithm>
6#include <cstring> 6#include <cstring>
7 7
8#define BOOST_HANA_CONFIG_ENABLE_STRING_UDL
9#include <boost/hana/string.hpp>
10#undef BOOST_HANA_CONFIG_ENABLE_STRING_UDL
11
8#include "common/assert.h" 12#include "common/assert.h"
9#include "common/atomic_ops.h" 13#include "common/atomic_ops.h"
10#include "common/common_types.h" 14#include "common/common_types.h"
@@ -19,6 +23,8 @@
19#include "core/memory.h" 23#include "core/memory.h"
20#include "video_core/gpu.h" 24#include "video_core/gpu.h"
21 25
26using namespace boost::hana::literals;
27
22namespace Core::Memory { 28namespace Core::Memory {
23 29
24// Implementation class used to keep the specifics of the memory subsystem hidden 30// Implementation class used to keep the specifics of the memory subsystem hidden
@@ -68,18 +74,6 @@ struct Memory::Impl {
68 return system.DeviceMemory().GetPointer(paddr) + vaddr; 74 return system.DeviceMemory().GetPointer(paddr) + vaddr;
69 } 75 }
70 76
71 [[nodiscard]] u8* GetPointer(const VAddr vaddr) const {
72 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
73 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
74 return pointer + vaddr;
75 }
76 const auto type = Common::PageTable::PageInfo::ExtractType(raw_pointer);
77 if (type == Common::PageType::RasterizerCachedMemory) {
78 return GetPointerFromRasterizerCachedMemory(vaddr);
79 }
80 return nullptr;
81 }
82
83 u8 Read8(const VAddr addr) { 77 u8 Read8(const VAddr addr) {
84 return Read<u8>(addr); 78 return Read<u8>(addr);
85 } 79 }
@@ -453,51 +447,106 @@ struct Memory::Impl {
453 } 447 }
454 448
455 /** 449 /**
456 * Reads a particular data type out of memory at the given virtual address. 450 * Returns a message like "Unmapped NameBits @ 0x{:016X}Suffix".
457 * 451 *
458 * @param vaddr The virtual address to read the data type from. 452 * @tparam NAME The caller name like "Read"_s or "Write"_s.
459 * 453 * @tparam BYTES The number of bits written. 0 is for read and sizeof(T) is for write.
460 * @tparam T The data type to read out of memory. This type *must* be 454 * @tparam SUFFIX A suffix. ""_s is for read and " = 0x{:016X}" is for write.
461 * trivially copyable, otherwise the behavior of this function
462 * is undefined.
463 *
464 * @returns The instance of T read from the specified virtual address.
465 */ 455 */
466 template <typename T> 456 template <boost::hana::string NAME, int BYTES, boost::hana::string SUFFIX>
467 T Read(VAddr vaddr) { 457 static consteval const char* GetPointerImplError() {
458 constexpr auto unmapped_fmt = ([]() {
459 constexpr auto prefix = "Unmapped "_s + NAME;
460 constexpr auto suffix = " @ 0x{:016X}"_s + SUFFIX;
461 const char* result = nullptr;
462 switch (BYTES * 8) {
463 case 0:
464 result = (prefix + suffix).c_str();
465 break;
466#define BITS_CASE(x) \
467 case x: \
468 result = (prefix + BOOST_HANA_STRING(#x) + suffix).c_str(); \
469 break;
470 BITS_CASE(8)
471 BITS_CASE(16)
472 BITS_CASE(32)
473 BITS_CASE(64)
474 BITS_CASE(128)
475#undef BITS_CASE
476 default:
477 break;
478 }
479 return result;
480 })();
481 static_assert(unmapped_fmt);
482 return unmapped_fmt;
483 }
484
485 [[nodiscard]] u8* GetPointerImpl(VAddr vaddr, auto on_unmapped, auto on_rasterizer) const {
468 // AARCH64 masks the upper 16 bit of all memory accesses 486 // AARCH64 masks the upper 16 bit of all memory accesses
469 vaddr &= 0xffffffffffffLL; 487 vaddr &= 0xffffffffffffLL;
470 488
471 if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { 489 if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) {
472 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); 490 on_unmapped();
473 return 0; 491 return nullptr;
474 } 492 }
475 493
476 // Avoid adding any extra logic to this fast-path block 494 // Avoid adding any extra logic to this fast-path block
477 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw(); 495 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
478 if (const u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) { 496 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
479 T value; 497 return &pointer[vaddr];
480 std::memcpy(&value, &pointer[vaddr], sizeof(T));
481 return value;
482 } 498 }
483 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) { 499 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
484 case Common::PageType::Unmapped: 500 case Common::PageType::Unmapped:
485 LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); 501 on_unmapped();
486 return 0; 502 return nullptr;
487 case Common::PageType::Memory: 503 case Common::PageType::Memory:
488 ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); 504 ASSERT_MSG(false, "Mapped memory page without a pointer @ 0x{:016X}", vaddr);
489 break; 505 return nullptr;
490 case Common::PageType::RasterizerCachedMemory: { 506 case Common::PageType::RasterizerCachedMemory: {
491 const u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)}; 507 u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
492 system.GPU().FlushRegion(vaddr, sizeof(T)); 508 on_rasterizer();
493 T value; 509 return host_ptr;
494 std::memcpy(&value, host_ptr, sizeof(T));
495 return value;
496 } 510 }
497 default: 511 default:
498 UNREACHABLE(); 512 UNREACHABLE();
499 } 513 }
500 return {}; 514 return nullptr;
515 }
516
517 [[nodiscard]] u8* GetPointer(const VAddr vaddr) const {
518 return GetPointerImpl(
519 vaddr,
520 [vaddr]() {
521 LOG_ERROR(HW_Memory, GetPointerImplError<"GetPointer"_s, 0, ""_s>(), vaddr);
522 },
523 []() {});
524 }
525
526 /**
527 * Reads a particular data type out of memory at the given virtual address.
528 *
529 * @param vaddr The virtual address to read the data type from.
530 *
531 * @tparam T The data type to read out of memory. This type *must* be
532 * trivially copyable, otherwise the behavior of this function
533 * is undefined.
534 *
535 * @returns The instance of T read from the specified virtual address.
536 */
537 template <typename T>
538 T Read(VAddr vaddr) {
539 T result = 0;
540 const u8* const ptr = GetPointerImpl(
541 vaddr,
542 [vaddr]() {
543 LOG_ERROR(HW_Memory, GetPointerImplError<"Read"_s, sizeof(T), ""_s>(), vaddr);
544 },
545 [&system = system, vaddr]() { system.GPU().FlushRegion(vaddr, sizeof(T)); });
546 if (ptr) {
547 std::memcpy(&result, ptr, sizeof(T));
548 }
549 return result;
501 } 550 }
502 551
503 /** 552 /**
@@ -511,110 +560,47 @@ struct Memory::Impl {
511 */ 560 */
512 template <typename T> 561 template <typename T>
513 void Write(VAddr vaddr, const T data) { 562 void Write(VAddr vaddr, const T data) {
514 // AARCH64 masks the upper 16 bit of all memory accesses 563 u8* const ptr = GetPointerImpl(
515 vaddr &= 0xffffffffffffLL; 564 vaddr,
516 565 [vaddr, data]() {
517 if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { 566 LOG_ERROR(HW_Memory, GetPointerImplError<"Write"_s, sizeof(T), " = 0x{:016X}"_s>(),
518 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 567 vaddr, static_cast<u64>(data));
519 static_cast<u32>(data), vaddr); 568 },
520 return; 569 [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); });
521 } 570 if (ptr) {
522 571 std::memcpy(ptr, &data, sizeof(T));
523 // Avoid adding any extra logic to this fast-path block
524 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
525 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
526 std::memcpy(&pointer[vaddr], &data, sizeof(T));
527 return;
528 }
529 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
530 case Common::PageType::Unmapped:
531 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
532 static_cast<u32>(data), vaddr);
533 return;
534 case Common::PageType::Memory:
535 ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
536 break;
537 case Common::PageType::RasterizerCachedMemory: {
538 u8* const host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
539 system.GPU().InvalidateRegion(vaddr, sizeof(T));
540 std::memcpy(host_ptr, &data, sizeof(T));
541 break;
542 }
543 default:
544 UNREACHABLE();
545 } 572 }
546 } 573 }
547 574
548 template <typename T> 575 template <typename T>
549 bool WriteExclusive(VAddr vaddr, const T data, const T expected) { 576 bool WriteExclusive(VAddr vaddr, const T data, const T expected) {
550 // AARCH64 masks the upper 16 bit of all memory accesses 577 u8* const ptr = GetPointerImpl(
551 vaddr &= 0xffffffffffffLL; 578 vaddr,
552 579 [vaddr, data]() {
553 if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { 580 LOG_ERROR(HW_Memory, GetPointerImplError<"Write"_s, sizeof(T), " = 0x{:016X}"_s>(),
554 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 581 vaddr, static_cast<u64>(data));
555 static_cast<u32>(data), vaddr); 582 },
556 return true; 583 [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); });
557 } 584 if (ptr) {
558 585 const auto volatile_pointer = reinterpret_cast<volatile T*>(ptr);
559 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw();
560 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
561 // NOTE: Avoid adding any extra logic to this fast-path block
562 const auto volatile_pointer = reinterpret_cast<volatile T*>(&pointer[vaddr]);
563 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); 586 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
564 } 587 }
565 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
566 case Common::PageType::Unmapped:
567 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
568 static_cast<u32>(data), vaddr);
569 return true;
570 case Common::PageType::Memory:
571 ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
572 break;
573 case Common::PageType::RasterizerCachedMemory: {
574 u8* host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
575 system.GPU().InvalidateRegion(vaddr, sizeof(T));
576 auto* pointer = reinterpret_cast<volatile T*>(&host_ptr);
577 return Common::AtomicCompareAndSwap(pointer, data, expected);
578 }
579 default:
580 UNREACHABLE();
581 }
582 return true; 588 return true;
583 } 589 }
584 590
585 bool WriteExclusive128(VAddr vaddr, const u128 data, const u128 expected) { 591 bool WriteExclusive128(VAddr vaddr, const u128 data, const u128 expected) {
586 // AARCH64 masks the upper 16 bit of all memory accesses 592 u8* const ptr = GetPointerImpl(
587 vaddr &= 0xffffffffffffLL; 593 vaddr,
588 594 [vaddr, data]() {
589 if (vaddr >= 1uLL << current_page_table->GetAddressSpaceBits()) { 595 LOG_ERROR(HW_Memory,
590 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, 596 GetPointerImplError<"Write"_s, sizeof(u128), " = 0x{:016X}{:016X}"_s>(),
591 static_cast<u32>(data[0]), vaddr); 597 vaddr, static_cast<u64>(data[1]), static_cast<u64>(data[0]));
592 return true; 598 },
593 } 599 [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(u128)); });
594 600 if (ptr) {
595 const uintptr_t raw_pointer = current_page_table->pointers[vaddr >> PAGE_BITS].Raw(); 601 const auto volatile_pointer = reinterpret_cast<volatile u64*>(ptr);
596 if (u8* const pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) {
597 // NOTE: Avoid adding any extra logic to this fast-path block
598 const auto volatile_pointer = reinterpret_cast<volatile u64*>(&pointer[vaddr]);
599 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); 602 return Common::AtomicCompareAndSwap(volatile_pointer, data, expected);
600 } 603 }
601 switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
602 case Common::PageType::Unmapped:
603 LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}{:016X}", sizeof(data) * 8,
604 static_cast<u64>(data[1]), static_cast<u64>(data[0]), vaddr);
605 return true;
606 case Common::PageType::Memory:
607 ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
608 break;
609 case Common::PageType::RasterizerCachedMemory: {
610 u8* host_ptr{GetPointerFromRasterizerCachedMemory(vaddr)};
611 system.GPU().InvalidateRegion(vaddr, sizeof(u128));
612 auto* pointer = reinterpret_cast<volatile u64*>(&host_ptr);
613 return Common::AtomicCompareAndSwap(pointer, data, expected);
614 }
615 default:
616 UNREACHABLE();
617 }
618 return true; 604 return true;
619 } 605 }
620 606