diff options
| author | 2022-09-09 21:38:28 -0700 | |
|---|---|---|
| committer | 2022-10-18 19:13:34 -0700 | |
| commit | ed591934fbffa32af0151302fd07e9fce776eb17 (patch) | |
| tree | d98a9c59b50ae3ad9d8d9e9bf118b5c19bb698bc | |
| parent | core: hle: kernel: k_memory_block: Update. (diff) | |
| download | yuzu-ed591934fbffa32af0151302fd07e9fce776eb17.tar.gz yuzu-ed591934fbffa32af0151302fd07e9fce776eb17.tar.xz yuzu-ed591934fbffa32af0151302fd07e9fce776eb17.zip | |
core: hle: kernel: k_page_table: Update, and integrate with new KMemoryBlockManager/SlabManager.
| -rw-r--r-- | src/core/hle/kernel/k_page_table.cpp | 619 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_page_table.h | 25 |
2 files changed, 393 insertions, 251 deletions
diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 8ebb75338..2cf46af0a 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp | |||
| @@ -49,6 +49,7 @@ KPageTable::~KPageTable() = default; | |||
| 49 | 49 | ||
| 50 | Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, | 50 | Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, |
| 51 | VAddr code_addr, std::size_t code_size, | 51 | VAddr code_addr, std::size_t code_size, |
| 52 | KMemoryBlockSlabManager* mem_block_slab_manager, | ||
| 52 | KMemoryManager::Pool pool) { | 53 | KMemoryManager::Pool pool) { |
| 53 | 54 | ||
| 54 | const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { | 55 | const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { |
| @@ -113,6 +114,7 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type | |||
| 113 | address_space_start = start; | 114 | address_space_start = start; |
| 114 | address_space_end = end; | 115 | address_space_end = end; |
| 115 | is_kernel = false; | 116 | is_kernel = false; |
| 117 | memory_block_slab_manager = mem_block_slab_manager; | ||
| 116 | 118 | ||
| 117 | // Determine the region we can place our undetermineds in | 119 | // Determine the region we can place our undetermineds in |
| 118 | VAddr alloc_start{}; | 120 | VAddr alloc_start{}; |
| @@ -254,7 +256,14 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type | |||
| 254 | 256 | ||
| 255 | page_table_impl.Resize(address_space_width, PageBits); | 257 | page_table_impl.Resize(address_space_width, PageBits); |
| 256 | 258 | ||
| 257 | return InitializeMemoryLayout(start, end); | 259 | return memory_block_manager.Initialize(address_space_start, address_space_end, |
| 260 | memory_block_slab_manager); | ||
| 261 | } | ||
| 262 | |||
| 263 | void KPageTable::Finalize() { | ||
| 264 | memory_block_manager.Finalize(memory_block_slab_manager, [&](VAddr addr, u64 size) { | ||
| 265 | system.Memory().UnmapRegion(page_table_impl, addr, size); | ||
| 266 | }); | ||
| 258 | } | 267 | } |
| 259 | 268 | ||
| 260 | Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, | 269 | Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, |
| @@ -271,6 +280,13 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat | |||
| 271 | R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, | 280 | R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, |
| 272 | KMemoryPermission::None, KMemoryPermission::None, | 281 | KMemoryPermission::None, KMemoryPermission::None, |
| 273 | KMemoryAttribute::None, KMemoryAttribute::None)); | 282 | KMemoryAttribute::None, KMemoryAttribute::None)); |
| 283 | |||
| 284 | // Create an update allocator. | ||
| 285 | Result allocator_result{ResultSuccess}; | ||
| 286 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 287 | memory_block_slab_manager); | ||
| 288 | |||
| 289 | // Allocate and open. | ||
| 274 | KPageGroup pg; | 290 | KPageGroup pg; |
| 275 | R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( | 291 | R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( |
| 276 | &pg, num_pages, | 292 | &pg, num_pages, |
| @@ -278,7 +294,10 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat | |||
| 278 | 294 | ||
| 279 | R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); | 295 | R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); |
| 280 | 296 | ||
| 281 | block_manager->Update(addr, num_pages, state, perm); | 297 | // Update the blocks. |
| 298 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, | ||
| 299 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, | ||
| 300 | KMemoryBlockDisableMergeAttribute::None); | ||
| 282 | 301 | ||
| 283 | return ResultSuccess; | 302 | return ResultSuccess; |
| 284 | } | 303 | } |
| @@ -307,6 +326,18 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size | |||
| 307 | KMemoryPermission::None, KMemoryAttribute::None, | 326 | KMemoryPermission::None, KMemoryAttribute::None, |
| 308 | KMemoryAttribute::None)); | 327 | KMemoryAttribute::None)); |
| 309 | 328 | ||
| 329 | // Create an update allocator for the source. | ||
| 330 | Result src_allocator_result{ResultSuccess}; | ||
| 331 | KMemoryBlockManagerUpdateAllocator src_allocator( | ||
| 332 | std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); | ||
| 333 | R_TRY(src_allocator_result); | ||
| 334 | |||
| 335 | // Create an update allocator for the destination. | ||
| 336 | Result dst_allocator_result{ResultSuccess}; | ||
| 337 | KMemoryBlockManagerUpdateAllocator dst_allocator( | ||
| 338 | std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); | ||
| 339 | R_TRY(dst_allocator_result); | ||
| 340 | |||
| 310 | // Map the code memory. | 341 | // Map the code memory. |
| 311 | { | 342 | { |
| 312 | // Determine the number of pages being operated on. | 343 | // Determine the number of pages being operated on. |
| @@ -335,10 +366,14 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size | |||
| 335 | unprot_guard.Cancel(); | 366 | unprot_guard.Cancel(); |
| 336 | 367 | ||
| 337 | // Apply the memory block updates. | 368 | // Apply the memory block updates. |
| 338 | block_manager->Update(src_address, num_pages, src_state, new_perm, | 369 | memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, |
| 339 | KMemoryAttribute::Locked); | 370 | src_state, new_perm, KMemoryAttribute::Locked, |
| 340 | block_manager->Update(dst_address, num_pages, KMemoryState::AliasCode, new_perm, | 371 | KMemoryBlockDisableMergeAttribute::Locked, |
| 341 | KMemoryAttribute::None); | 372 | KMemoryBlockDisableMergeAttribute::None); |
| 373 | memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, | ||
| 374 | KMemoryState::AliasCode, new_perm, KMemoryAttribute::None, | ||
| 375 | KMemoryBlockDisableMergeAttribute::Normal, | ||
| 376 | KMemoryBlockDisableMergeAttribute::None); | ||
| 342 | } | 377 | } |
| 343 | 378 | ||
| 344 | return ResultSuccess; | 379 | return ResultSuccess; |
| @@ -370,7 +405,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si | |||
| 370 | // Determine whether any pages being unmapped are code. | 405 | // Determine whether any pages being unmapped are code. |
| 371 | bool any_code_pages = false; | 406 | bool any_code_pages = false; |
| 372 | { | 407 | { |
| 373 | KMemoryBlockManager::const_iterator it = block_manager->FindIterator(dst_address); | 408 | KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(dst_address); |
| 374 | while (true) { | 409 | while (true) { |
| 375 | // Get the memory info. | 410 | // Get the memory info. |
| 376 | const KMemoryInfo info = it->GetMemoryInfo(); | 411 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -408,6 +443,20 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si | |||
| 408 | // Determine the number of pages being operated on. | 443 | // Determine the number of pages being operated on. |
| 409 | const std::size_t num_pages = size / PageSize; | 444 | const std::size_t num_pages = size / PageSize; |
| 410 | 445 | ||
| 446 | // Create an update allocator for the source. | ||
| 447 | Result src_allocator_result{ResultSuccess}; | ||
| 448 | KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), | ||
| 449 | memory_block_slab_manager, | ||
| 450 | num_src_allocator_blocks); | ||
| 451 | R_TRY(src_allocator_result); | ||
| 452 | |||
| 453 | // Create an update allocator for the destination. | ||
| 454 | Result dst_allocator_result{ResultSuccess}; | ||
| 455 | KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), | ||
| 456 | memory_block_slab_manager, | ||
| 457 | num_dst_allocator_blocks); | ||
| 458 | R_TRY(dst_allocator_result); | ||
| 459 | |||
| 411 | // Unmap the aliased copy of the pages. | 460 | // Unmap the aliased copy of the pages. |
| 412 | R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); | 461 | R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); |
| 413 | 462 | ||
| @@ -416,9 +465,14 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si | |||
| 416 | OperationType::ChangePermissions)); | 465 | OperationType::ChangePermissions)); |
| 417 | 466 | ||
| 418 | // Apply the memory block updates. | 467 | // Apply the memory block updates. |
| 419 | block_manager->Update(dst_address, num_pages, KMemoryState::None); | 468 | memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, |
| 420 | block_manager->Update(src_address, num_pages, KMemoryState::Normal, | 469 | KMemoryState::None, KMemoryPermission::None, |
| 421 | KMemoryPermission::UserReadWrite); | 470 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, |
| 471 | KMemoryBlockDisableMergeAttribute::Normal); | ||
| 472 | memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, | ||
| 473 | KMemoryState::Normal, KMemoryPermission::UserReadWrite, | ||
| 474 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, | ||
| 475 | KMemoryBlockDisableMergeAttribute::Locked); | ||
| 422 | 476 | ||
| 423 | // Note that we reprotected pages. | 477 | // Note that we reprotected pages. |
| 424 | reprotected_pages = true; | 478 | reprotected_pages = true; |
| @@ -434,55 +488,12 @@ VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, | |||
| 434 | 488 | ||
| 435 | if (num_pages <= region_num_pages) { | 489 | if (num_pages <= region_num_pages) { |
| 436 | if (this->IsAslrEnabled()) { | 490 | if (this->IsAslrEnabled()) { |
| 437 | // Try to directly find a free area up to 8 times. | 491 | UNIMPLEMENTED(); |
| 438 | for (std::size_t i = 0; i < 8; i++) { | ||
| 439 | const std::size_t random_offset = | ||
| 440 | KSystemControl::GenerateRandomRange( | ||
| 441 | 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * | ||
| 442 | alignment; | ||
| 443 | const VAddr candidate = | ||
| 444 | Common::AlignDown((region_start + random_offset), alignment) + offset; | ||
| 445 | |||
| 446 | KMemoryInfo info = this->QueryInfoImpl(candidate); | ||
| 447 | |||
| 448 | if (info.state != KMemoryState::Free) { | ||
| 449 | continue; | ||
| 450 | } | ||
| 451 | if (region_start > candidate) { | ||
| 452 | continue; | ||
| 453 | } | ||
| 454 | if (info.GetAddress() + guard_pages * PageSize > candidate) { | ||
| 455 | continue; | ||
| 456 | } | ||
| 457 | |||
| 458 | const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; | ||
| 459 | if (candidate_end > info.GetLastAddress()) { | ||
| 460 | continue; | ||
| 461 | } | ||
| 462 | if (candidate_end > region_start + region_num_pages * PageSize - 1) { | ||
| 463 | continue; | ||
| 464 | } | ||
| 465 | |||
| 466 | address = candidate; | ||
| 467 | break; | ||
| 468 | } | ||
| 469 | // Fall back to finding the first free area with a random offset. | ||
| 470 | if (address == 0) { | ||
| 471 | // NOTE: Nintendo does not account for guard pages here. | ||
| 472 | // This may theoretically cause an offset to be chosen that cannot be mapped. We | ||
| 473 | // will account for guard pages. | ||
| 474 | const std::size_t offset_pages = KSystemControl::GenerateRandomRange( | ||
| 475 | 0, region_num_pages - num_pages - guard_pages); | ||
| 476 | address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, | ||
| 477 | region_num_pages - offset_pages, num_pages, | ||
| 478 | alignment, offset, guard_pages); | ||
| 479 | } | ||
| 480 | } | 492 | } |
| 481 | |||
| 482 | // Find the first free area. | 493 | // Find the first free area. |
| 483 | if (address == 0) { | 494 | if (address == 0) { |
| 484 | address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, | 495 | address = memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, |
| 485 | alignment, offset, guard_pages); | 496 | alignment, offset, guard_pages); |
| 486 | } | 497 | } |
| 487 | } | 498 | } |
| 488 | 499 | ||
| @@ -649,11 +660,19 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTab | |||
| 649 | KMemoryPermission::None, KMemoryAttribute::All, | 660 | KMemoryPermission::None, KMemoryAttribute::All, |
| 650 | KMemoryAttribute::None)); | 661 | KMemoryAttribute::None)); |
| 651 | 662 | ||
| 663 | // Create an update allocator. | ||
| 664 | Result allocator_result{ResultSuccess}; | ||
| 665 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 666 | memory_block_slab_manager, num_allocator_blocks); | ||
| 667 | R_TRY(allocator_result); | ||
| 668 | |||
| 652 | CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); | 669 | CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); |
| 653 | 670 | ||
| 654 | // Apply the memory block update. | 671 | // Apply the memory block update. |
| 655 | block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, | 672 | memory_block_manager.Update(std::addressof(allocator), dst_addr, num_pages, KMemoryState::Free, |
| 656 | KMemoryAttribute::None); | 673 | KMemoryPermission::None, KMemoryAttribute::None, |
| 674 | KMemoryBlockDisableMergeAttribute::None, | ||
| 675 | KMemoryBlockDisableMergeAttribute::Normal); | ||
| 657 | 676 | ||
| 658 | system.InvalidateCpuInstructionCaches(); | 677 | system.InvalidateCpuInstructionCaches(); |
| 659 | 678 | ||
| @@ -682,10 +701,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 682 | cur_address = address; | 701 | cur_address = address; |
| 683 | mapped_size = 0; | 702 | mapped_size = 0; |
| 684 | 703 | ||
| 685 | auto it = block_manager->FindIterator(cur_address); | 704 | auto it = memory_block_manager.FindIterator(cur_address); |
| 686 | while (true) { | 705 | while (true) { |
| 687 | // Check that the iterator is valid. | 706 | // Check that the iterator is valid. |
| 688 | ASSERT(it != block_manager->end()); | 707 | ASSERT(it != memory_block_manager.end()); |
| 689 | 708 | ||
| 690 | // Get the memory info. | 709 | // Get the memory info. |
| 691 | const KMemoryInfo info = it->GetMemoryInfo(); | 710 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -739,10 +758,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 739 | size_t checked_mapped_size = 0; | 758 | size_t checked_mapped_size = 0; |
| 740 | cur_address = address; | 759 | cur_address = address; |
| 741 | 760 | ||
| 742 | auto it = block_manager->FindIterator(cur_address); | 761 | auto it = memory_block_manager.FindIterator(cur_address); |
| 743 | while (true) { | 762 | while (true) { |
| 744 | // Check that the iterator is valid. | 763 | // Check that the iterator is valid. |
| 745 | ASSERT(it != block_manager->end()); | 764 | ASSERT(it != memory_block_manager.end()); |
| 746 | 765 | ||
| 747 | // Get the memory info. | 766 | // Get the memory info. |
| 748 | const KMemoryInfo info = it->GetMemoryInfo(); | 767 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -782,6 +801,14 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 782 | } | 801 | } |
| 783 | } | 802 | } |
| 784 | 803 | ||
| 804 | // Create an update allocator. | ||
| 805 | ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); | ||
| 806 | Result allocator_result{ResultSuccess}; | ||
| 807 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 808 | memory_block_slab_manager, | ||
| 809 | num_allocator_blocks); | ||
| 810 | R_TRY(allocator_result); | ||
| 811 | |||
| 785 | // Reset the current tracking address, and make sure we clean up on failure. | 812 | // Reset the current tracking address, and make sure we clean up on failure. |
| 786 | cur_address = address; | 813 | cur_address = address; |
| 787 | auto unmap_guard = detail::ScopeExit([&] { | 814 | auto unmap_guard = detail::ScopeExit([&] { |
| @@ -791,10 +818,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 791 | // Iterate, unmapping the pages. | 818 | // Iterate, unmapping the pages. |
| 792 | cur_address = address; | 819 | cur_address = address; |
| 793 | 820 | ||
| 794 | auto it = block_manager->FindIterator(cur_address); | 821 | auto it = memory_block_manager.FindIterator(cur_address); |
| 795 | while (true) { | 822 | while (true) { |
| 796 | // Check that the iterator is valid. | 823 | // Check that the iterator is valid. |
| 797 | ASSERT(it != block_manager->end()); | 824 | ASSERT(it != memory_block_manager.end()); |
| 798 | 825 | ||
| 799 | // Get the memory info. | 826 | // Get the memory info. |
| 800 | const KMemoryInfo info = it->GetMemoryInfo(); | 827 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -830,10 +857,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 830 | PAddr pg_phys_addr = pg_it->GetAddress(); | 857 | PAddr pg_phys_addr = pg_it->GetAddress(); |
| 831 | size_t pg_pages = pg_it->GetNumPages(); | 858 | size_t pg_pages = pg_it->GetNumPages(); |
| 832 | 859 | ||
| 833 | auto it = block_manager->FindIterator(cur_address); | 860 | auto it = memory_block_manager.FindIterator(cur_address); |
| 834 | while (true) { | 861 | while (true) { |
| 835 | // Check that the iterator is valid. | 862 | // Check that the iterator is valid. |
| 836 | ASSERT(it != block_manager->end()); | 863 | ASSERT(it != memory_block_manager.end()); |
| 837 | 864 | ||
| 838 | // Get the memory info. | 865 | // Get the memory info. |
| 839 | const KMemoryInfo info = it->GetMemoryInfo(); | 866 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -889,10 +916,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 889 | mapped_physical_memory_size += (size - mapped_size); | 916 | mapped_physical_memory_size += (size - mapped_size); |
| 890 | 917 | ||
| 891 | // Update the relevant memory blocks. | 918 | // Update the relevant memory blocks. |
| 892 | block_manager->Update(address, size / PageSize, KMemoryState::Free, | 919 | memory_block_manager.UpdateIfMatch( |
| 893 | KMemoryPermission::None, KMemoryAttribute::None, | 920 | std::addressof(allocator), address, size / PageSize, KMemoryState::Free, |
| 894 | KMemoryState::Normal, KMemoryPermission::UserReadWrite, | 921 | KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal, |
| 895 | KMemoryAttribute::None); | 922 | KMemoryPermission::UserReadWrite, KMemoryAttribute::None); |
| 896 | 923 | ||
| 897 | // Cancel our guard. | 924 | // Cancel our guard. |
| 898 | unmap_guard.Cancel(); | 925 | unmap_guard.Cancel(); |
| @@ -924,10 +951,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 924 | cur_address = address; | 951 | cur_address = address; |
| 925 | mapped_size = 0; | 952 | mapped_size = 0; |
| 926 | 953 | ||
| 927 | auto it = block_manager->FindIterator(cur_address); | 954 | auto it = memory_block_manager.FindIterator(cur_address); |
| 928 | while (true) { | 955 | while (true) { |
| 929 | // Check that the iterator is valid. | 956 | // Check that the iterator is valid. |
| 930 | ASSERT(it != block_manager->end()); | 957 | ASSERT(it != memory_block_manager.end()); |
| 931 | 958 | ||
| 932 | // Get the memory info. | 959 | // Get the memory info. |
| 933 | const KMemoryInfo info = it->GetMemoryInfo(); | 960 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -1022,6 +1049,13 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 1022 | } | 1049 | } |
| 1023 | ASSERT(pg.GetNumPages() == mapped_size / PageSize); | 1050 | ASSERT(pg.GetNumPages() == mapped_size / PageSize); |
| 1024 | 1051 | ||
| 1052 | // Create an update allocator. | ||
| 1053 | ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); | ||
| 1054 | Result allocator_result{ResultSuccess}; | ||
| 1055 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1056 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1057 | R_TRY(allocator_result); | ||
| 1058 | |||
| 1025 | // Reset the current tracking address, and make sure we clean up on failure. | 1059 | // Reset the current tracking address, and make sure we clean up on failure. |
| 1026 | cur_address = address; | 1060 | cur_address = address; |
| 1027 | auto remap_guard = detail::ScopeExit([&] { | 1061 | auto remap_guard = detail::ScopeExit([&] { |
| @@ -1030,7 +1064,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 1030 | cur_address = address; | 1064 | cur_address = address; |
| 1031 | 1065 | ||
| 1032 | // Iterate over the memory we unmapped. | 1066 | // Iterate over the memory we unmapped. |
| 1033 | auto it = block_manager->FindIterator(cur_address); | 1067 | auto it = memory_block_manager.FindIterator(cur_address); |
| 1034 | auto pg_it = pg.Nodes().begin(); | 1068 | auto pg_it = pg.Nodes().begin(); |
| 1035 | PAddr pg_phys_addr = pg_it->GetAddress(); | 1069 | PAddr pg_phys_addr = pg_it->GetAddress(); |
| 1036 | size_t pg_pages = pg_it->GetNumPages(); | 1070 | size_t pg_pages = pg_it->GetNumPages(); |
| @@ -1085,10 +1119,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 1085 | }); | 1119 | }); |
| 1086 | 1120 | ||
| 1087 | // Iterate over the memory, unmapping as we go. | 1121 | // Iterate over the memory, unmapping as we go. |
| 1088 | auto it = block_manager->FindIterator(cur_address); | 1122 | auto it = memory_block_manager.FindIterator(cur_address); |
| 1089 | while (true) { | 1123 | while (true) { |
| 1090 | // Check that the iterator is valid. | 1124 | // Check that the iterator is valid. |
| 1091 | ASSERT(it != block_manager->end()); | 1125 | ASSERT(it != memory_block_manager.end()); |
| 1092 | 1126 | ||
| 1093 | // Get the memory info. | 1127 | // Get the memory info. |
| 1094 | const KMemoryInfo info = it->GetMemoryInfo(); | 1128 | const KMemoryInfo info = it->GetMemoryInfo(); |
| @@ -1120,8 +1154,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 1120 | process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); | 1154 | process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); |
| 1121 | 1155 | ||
| 1122 | // Update memory blocks. | 1156 | // Update memory blocks. |
| 1123 | block_manager->Update(address, size / PageSize, KMemoryState::Free, KMemoryPermission::None, | 1157 | memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, |
| 1124 | KMemoryAttribute::None); | 1158 | KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None, |
| 1159 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1160 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1125 | 1161 | ||
| 1126 | // TODO(bunnei): This is a workaround until the next set of changes, where we add reference | 1162 | // TODO(bunnei): This is a workaround until the next set of changes, where we add reference |
| 1127 | // counting for mapped pages. Until then, we must manually close the reference to the page | 1163 | // counting for mapped pages. Until then, we must manually close the reference to the page |
| @@ -1134,83 +1170,134 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { | |||
| 1134 | return ResultSuccess; | 1170 | return ResultSuccess; |
| 1135 | } | 1171 | } |
| 1136 | 1172 | ||
| 1137 | Result KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { | 1173 | Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { |
| 1174 | // Lock the table. | ||
| 1138 | KScopedLightLock lk(general_lock); | 1175 | KScopedLightLock lk(general_lock); |
| 1139 | 1176 | ||
| 1140 | KMemoryState src_state{}; | 1177 | // Validate that the source address's state is valid. |
| 1141 | CASCADE_CODE(CheckMemoryState( | 1178 | KMemoryState src_state; |
| 1142 | &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, | 1179 | size_t num_src_allocator_blocks; |
| 1143 | KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::UserReadWrite, | 1180 | R_TRY(this->CheckMemoryState(std::addressof(src_state), nullptr, nullptr, |
| 1144 | KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); | 1181 | std::addressof(num_src_allocator_blocks), src_address, size, |
| 1182 | KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, | ||
| 1183 | KMemoryPermission::All, KMemoryPermission::UserReadWrite, | ||
| 1184 | KMemoryAttribute::All, KMemoryAttribute::None)); | ||
| 1145 | 1185 | ||
| 1146 | if (IsRegionMapped(dst_addr, size)) { | 1186 | // Validate that the dst address's state is valid. |
| 1147 | return ResultInvalidCurrentMemory; | 1187 | size_t num_dst_allocator_blocks; |
| 1148 | } | 1188 | R_TRY(this->CheckMemoryState(std::addressof(num_dst_allocator_blocks), dst_address, size, |
| 1189 | KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, | ||
| 1190 | KMemoryPermission::None, KMemoryAttribute::None, | ||
| 1191 | KMemoryAttribute::None)); | ||
| 1149 | 1192 | ||
| 1193 | // Create an update allocator for the source. | ||
| 1194 | Result src_allocator_result{ResultSuccess}; | ||
| 1195 | KMemoryBlockManagerUpdateAllocator src_allocator( | ||
| 1196 | std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); | ||
| 1197 | R_TRY(src_allocator_result); | ||
| 1198 | |||
| 1199 | // Create an update allocator for the destination. | ||
| 1200 | Result dst_allocator_result{ResultSuccess}; | ||
| 1201 | KMemoryBlockManagerUpdateAllocator dst_allocator( | ||
| 1202 | std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); | ||
| 1203 | R_TRY(dst_allocator_result); | ||
| 1204 | |||
| 1205 | // Map the memory. | ||
| 1150 | KPageGroup page_linked_list; | 1206 | KPageGroup page_linked_list; |
| 1151 | const std::size_t num_pages{size / PageSize}; | 1207 | const std::size_t num_pages{size / PageSize}; |
| 1208 | const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( | ||
| 1209 | KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); | ||
| 1210 | const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; | ||
| 1152 | 1211 | ||
| 1153 | AddRegionToPages(src_addr, num_pages, page_linked_list); | 1212 | AddRegionToPages(src_address, num_pages, page_linked_list); |
| 1154 | |||
| 1155 | { | 1213 | { |
| 1214 | // Reprotect the source as kernel-read/not mapped. | ||
| 1156 | auto block_guard = detail::ScopeExit([&] { | 1215 | auto block_guard = detail::ScopeExit([&] { |
| 1157 | Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, | 1216 | Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, |
| 1158 | OperationType::ChangePermissions); | 1217 | OperationType::ChangePermissions); |
| 1159 | }); | 1218 | }); |
| 1160 | 1219 | R_TRY(Operate(src_address, num_pages, new_src_perm, OperationType::ChangePermissions)); | |
| 1161 | CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, | 1220 | R_TRY(MapPages(dst_address, page_linked_list, KMemoryPermission::UserReadWrite)); |
| 1162 | OperationType::ChangePermissions)); | ||
| 1163 | CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::UserReadWrite)); | ||
| 1164 | 1221 | ||
| 1165 | block_guard.Cancel(); | 1222 | block_guard.Cancel(); |
| 1166 | } | 1223 | } |
| 1167 | 1224 | ||
| 1168 | block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None, | 1225 | // Apply the memory block updates. |
| 1169 | KMemoryAttribute::Locked); | 1226 | memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, |
| 1170 | block_manager->Update(dst_addr, num_pages, KMemoryState::Stack, | 1227 | new_src_perm, new_src_attr, |
| 1171 | KMemoryPermission::UserReadWrite); | 1228 | KMemoryBlockDisableMergeAttribute::Locked, |
| 1229 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1230 | memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, | ||
| 1231 | KMemoryState::Stack, KMemoryPermission::UserReadWrite, | ||
| 1232 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, | ||
| 1233 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1172 | 1234 | ||
| 1173 | return ResultSuccess; | 1235 | return ResultSuccess; |
| 1174 | } | 1236 | } |
| 1175 | 1237 | ||
| 1176 | Result KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { | 1238 | Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { |
| 1239 | // Lock the table. | ||
| 1177 | KScopedLightLock lk(general_lock); | 1240 | KScopedLightLock lk(general_lock); |
| 1178 | 1241 | ||
| 1179 | KMemoryState src_state{}; | 1242 | // Validate that the source address's state is valid. |
| 1180 | CASCADE_CODE(CheckMemoryState( | 1243 | KMemoryState src_state; |
| 1181 | &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, | 1244 | size_t num_src_allocator_blocks; |
| 1182 | KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::None, | 1245 | R_TRY(this->CheckMemoryState( |
| 1183 | KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); | 1246 | std::addressof(src_state), nullptr, nullptr, std::addressof(num_src_allocator_blocks), |
| 1247 | src_address, size, KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, | ||
| 1248 | KMemoryPermission::All, KMemoryPermission::NotMapped | KMemoryPermission::KernelRead, | ||
| 1249 | KMemoryAttribute::All, KMemoryAttribute::Locked)); | ||
| 1250 | |||
| 1251 | // Validate that the dst address's state is valid. | ||
| 1252 | KMemoryPermission dst_perm; | ||
| 1253 | size_t num_dst_allocator_blocks; | ||
| 1254 | R_TRY(this->CheckMemoryState( | ||
| 1255 | nullptr, std::addressof(dst_perm), nullptr, std::addressof(num_dst_allocator_blocks), | ||
| 1256 | dst_address, size, KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, | ||
| 1257 | KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); | ||
| 1258 | |||
| 1259 | // Create an update allocator for the source. | ||
| 1260 | Result src_allocator_result{ResultSuccess}; | ||
| 1261 | KMemoryBlockManagerUpdateAllocator src_allocator( | ||
| 1262 | std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); | ||
| 1263 | R_TRY(src_allocator_result); | ||
| 1184 | 1264 | ||
| 1185 | KMemoryPermission dst_perm{}; | 1265 | // Create an update allocator for the destination. |
| 1186 | CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, nullptr, dst_addr, size, | 1266 | Result dst_allocator_result{ResultSuccess}; |
| 1187 | KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, | 1267 | KMemoryBlockManagerUpdateAllocator dst_allocator( |
| 1188 | KMemoryPermission::None, KMemoryAttribute::Mask, | 1268 | std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); |
| 1189 | KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); | 1269 | R_TRY(dst_allocator_result); |
| 1190 | 1270 | ||
| 1191 | KPageGroup src_pages; | 1271 | KPageGroup src_pages; |
| 1192 | KPageGroup dst_pages; | 1272 | KPageGroup dst_pages; |
| 1193 | const std::size_t num_pages{size / PageSize}; | 1273 | const std::size_t num_pages{size / PageSize}; |
| 1194 | 1274 | ||
| 1195 | AddRegionToPages(src_addr, num_pages, src_pages); | 1275 | AddRegionToPages(src_address, num_pages, src_pages); |
| 1196 | AddRegionToPages(dst_addr, num_pages, dst_pages); | 1276 | AddRegionToPages(dst_address, num_pages, dst_pages); |
| 1197 | 1277 | ||
| 1198 | if (!dst_pages.IsEqual(src_pages)) { | 1278 | if (!dst_pages.IsEqual(src_pages)) { |
| 1199 | return ResultInvalidMemoryRegion; | 1279 | return ResultInvalidMemoryRegion; |
| 1200 | } | 1280 | } |
| 1201 | 1281 | ||
| 1202 | { | 1282 | { |
| 1203 | auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); | 1283 | auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); |
| 1204 | 1284 | ||
| 1205 | CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); | 1285 | R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); |
| 1206 | CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, | 1286 | R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, |
| 1207 | OperationType::ChangePermissions)); | 1287 | OperationType::ChangePermissions)); |
| 1208 | 1288 | ||
| 1209 | block_guard.Cancel(); | 1289 | block_guard.Cancel(); |
| 1210 | } | 1290 | } |
| 1211 | 1291 | ||
| 1212 | block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::UserReadWrite); | 1292 | // Apply the memory block updates. |
| 1213 | block_manager->Update(dst_addr, num_pages, KMemoryState::Free); | 1293 | memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, |
| 1294 | KMemoryPermission::UserReadWrite, KMemoryAttribute::None, | ||
| 1295 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1296 | KMemoryBlockDisableMergeAttribute::Locked); | ||
| 1297 | memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, | ||
| 1298 | KMemoryState::None, KMemoryPermission::None, KMemoryAttribute::None, | ||
| 1299 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1300 | KMemoryBlockDisableMergeAttribute::Normal); | ||
| 1214 | 1301 | ||
| 1215 | return ResultSuccess; | 1302 | return ResultSuccess; |
| 1216 | } | 1303 | } |
| @@ -1254,11 +1341,18 @@ Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemory | |||
| 1254 | KMemoryPermission::None, KMemoryPermission::None, | 1341 | KMemoryPermission::None, KMemoryPermission::None, |
| 1255 | KMemoryAttribute::None, KMemoryAttribute::None)); | 1342 | KMemoryAttribute::None, KMemoryAttribute::None)); |
| 1256 | 1343 | ||
| 1344 | // Create an update allocator. | ||
| 1345 | Result allocator_result{ResultSuccess}; | ||
| 1346 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1347 | memory_block_slab_manager); | ||
| 1348 | |||
| 1257 | // Map the pages. | 1349 | // Map the pages. |
| 1258 | R_TRY(MapPages(address, page_linked_list, perm)); | 1350 | R_TRY(MapPages(address, page_linked_list, perm)); |
| 1259 | 1351 | ||
| 1260 | // Update the blocks. | 1352 | // Update the blocks. |
| 1261 | block_manager->Update(address, num_pages, state, perm); | 1353 | memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, |
| 1354 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, | ||
| 1355 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1262 | 1356 | ||
| 1263 | return ResultSuccess; | 1357 | return ResultSuccess; |
| 1264 | } | 1358 | } |
| @@ -1288,6 +1382,11 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t | |||
| 1288 | KMemoryAttribute::None, KMemoryAttribute::None) | 1382 | KMemoryAttribute::None, KMemoryAttribute::None) |
| 1289 | .IsSuccess()); | 1383 | .IsSuccess()); |
| 1290 | 1384 | ||
| 1385 | // Create an update allocator. | ||
| 1386 | Result allocator_result{ResultSuccess}; | ||
| 1387 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1388 | memory_block_slab_manager); | ||
| 1389 | |||
| 1291 | // Perform mapping operation. | 1390 | // Perform mapping operation. |
| 1292 | if (is_pa_valid) { | 1391 | if (is_pa_valid) { |
| 1293 | R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); | 1392 | R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); |
| @@ -1296,7 +1395,9 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t | |||
| 1296 | } | 1395 | } |
| 1297 | 1396 | ||
| 1298 | // Update the blocks. | 1397 | // Update the blocks. |
| 1299 | block_manager->Update(addr, num_pages, state, perm); | 1398 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, |
| 1399 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, | ||
| 1400 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1300 | 1401 | ||
| 1301 | // We successfully mapped the pages. | 1402 | // We successfully mapped the pages. |
| 1302 | *out_addr = addr; | 1403 | *out_addr = addr; |
| @@ -1321,25 +1422,36 @@ Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { | |||
| 1321 | return ResultSuccess; | 1422 | return ResultSuccess; |
| 1322 | } | 1423 | } |
| 1323 | 1424 | ||
| 1324 | Result KPageTable::UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state) { | 1425 | Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { |
| 1325 | // Check that the unmap is in range. | 1426 | // Check that the unmap is in range. |
| 1326 | const std::size_t num_pages{page_linked_list.GetNumPages()}; | 1427 | const std::size_t num_pages{page_linked_list.GetNumPages()}; |
| 1327 | const std::size_t size{num_pages * PageSize}; | 1428 | const std::size_t size{num_pages * PageSize}; |
| 1328 | R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); | 1429 | R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); |
| 1329 | 1430 | ||
| 1330 | // Lock the table. | 1431 | // Lock the table. |
| 1331 | KScopedLightLock lk(general_lock); | 1432 | KScopedLightLock lk(general_lock); |
| 1332 | 1433 | ||
| 1333 | // Check the memory state. | 1434 | // Check the memory state. |
| 1334 | R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, state, KMemoryPermission::None, | 1435 | size_t num_allocator_blocks; |
| 1436 | R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, | ||
| 1437 | KMemoryState::All, state, KMemoryPermission::None, | ||
| 1335 | KMemoryPermission::None, KMemoryAttribute::All, | 1438 | KMemoryPermission::None, KMemoryAttribute::All, |
| 1336 | KMemoryAttribute::None)); | 1439 | KMemoryAttribute::None)); |
| 1337 | 1440 | ||
| 1441 | // Create an update allocator. | ||
| 1442 | Result allocator_result{ResultSuccess}; | ||
| 1443 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1444 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1445 | R_TRY(allocator_result); | ||
| 1446 | |||
| 1338 | // Perform the unmap. | 1447 | // Perform the unmap. |
| 1339 | R_TRY(UnmapPages(addr, page_linked_list)); | 1448 | R_TRY(UnmapPages(address, page_linked_list)); |
| 1340 | 1449 | ||
| 1341 | // Update the blocks. | 1450 | // Update the blocks. |
| 1342 | block_manager->Update(addr, num_pages, state, KMemoryPermission::None); | 1451 | memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, |
| 1452 | KMemoryPermission::None, KMemoryAttribute::None, | ||
| 1453 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1454 | KMemoryBlockDisableMergeAttribute::Normal); | ||
| 1343 | 1455 | ||
| 1344 | return ResultSuccess; | 1456 | return ResultSuccess; |
| 1345 | } | 1457 | } |
| @@ -1359,11 +1471,20 @@ Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState | |||
| 1359 | KMemoryPermission::None, KMemoryAttribute::All, | 1471 | KMemoryPermission::None, KMemoryAttribute::All, |
| 1360 | KMemoryAttribute::None)); | 1472 | KMemoryAttribute::None)); |
| 1361 | 1473 | ||
| 1474 | // Create an update allocator. | ||
| 1475 | Result allocator_result{ResultSuccess}; | ||
| 1476 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1477 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1478 | R_TRY(allocator_result); | ||
| 1479 | |||
| 1362 | // Perform the unmap. | 1480 | // Perform the unmap. |
| 1363 | R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); | 1481 | R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); |
| 1364 | 1482 | ||
| 1365 | // Update the blocks. | 1483 | // Update the blocks. |
| 1366 | block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); | 1484 | memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, |
| 1485 | KMemoryPermission::None, KMemoryAttribute::None, | ||
| 1486 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1487 | KMemoryBlockDisableMergeAttribute::Normal); | ||
| 1367 | 1488 | ||
| 1368 | return ResultSuccess; | 1489 | return ResultSuccess; |
| 1369 | } | 1490 | } |
| @@ -1435,13 +1556,21 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, | |||
| 1435 | // Succeed if there's nothing to do. | 1556 | // Succeed if there's nothing to do. |
| 1436 | R_SUCCEED_IF(old_perm == new_perm && old_state == new_state); | 1557 | R_SUCCEED_IF(old_perm == new_perm && old_state == new_state); |
| 1437 | 1558 | ||
| 1559 | // Create an update allocator. | ||
| 1560 | Result allocator_result{ResultSuccess}; | ||
| 1561 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1562 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1563 | R_TRY(allocator_result); | ||
| 1564 | |||
| 1438 | // Perform mapping operation. | 1565 | // Perform mapping operation. |
| 1439 | const auto operation = | 1566 | const auto operation = |
| 1440 | was_x ? OperationType::ChangePermissionsAndRefresh : OperationType::ChangePermissions; | 1567 | was_x ? OperationType::ChangePermissionsAndRefresh : OperationType::ChangePermissions; |
| 1441 | R_TRY(Operate(addr, num_pages, new_perm, operation)); | 1568 | R_TRY(Operate(addr, num_pages, new_perm, operation)); |
| 1442 | 1569 | ||
| 1443 | // Update the blocks. | 1570 | // Update the blocks. |
| 1444 | block_manager->Update(addr, num_pages, new_state, new_perm, KMemoryAttribute::None); | 1571 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, new_state, new_perm, |
| 1572 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, | ||
| 1573 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1445 | 1574 | ||
| 1446 | // Ensure cache coherency, if we're setting pages as executable. | 1575 | // Ensure cache coherency, if we're setting pages as executable. |
| 1447 | if (is_x) { | 1576 | if (is_x) { |
| @@ -1454,51 +1583,30 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, | |||
| 1454 | KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { | 1583 | KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { |
| 1455 | KScopedLightLock lk(general_lock); | 1584 | KScopedLightLock lk(general_lock); |
| 1456 | 1585 | ||
| 1457 | return block_manager->FindBlock(addr).GetMemoryInfo(); | 1586 | return memory_block_manager.FindBlock(addr)->GetMemoryInfo(); |
| 1458 | } | 1587 | } |
| 1459 | 1588 | ||
| 1460 | KMemoryInfo KPageTable::QueryInfo(VAddr addr) { | 1589 | KMemoryInfo KPageTable::QueryInfo(VAddr addr) { |
| 1461 | if (!Contains(addr, 1)) { | 1590 | if (!Contains(addr, 1)) { |
| 1462 | return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible, | 1591 | return { |
| 1463 | KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None}; | 1592 | .m_address = address_space_end, |
| 1593 | .m_size = 0 - address_space_end, | ||
| 1594 | .m_state = static_cast<KMemoryState>(Svc::MemoryState::Inaccessible), | ||
| 1595 | .m_device_disable_merge_left_count = 0, | ||
| 1596 | .m_device_disable_merge_right_count = 0, | ||
| 1597 | .m_ipc_lock_count = 0, | ||
| 1598 | .m_device_use_count = 0, | ||
| 1599 | .m_ipc_disable_merge_count = 0, | ||
| 1600 | .m_permission = KMemoryPermission::None, | ||
| 1601 | .m_attribute = KMemoryAttribute::None, | ||
| 1602 | .m_original_permission = KMemoryPermission::None, | ||
| 1603 | .m_disable_merge_attribute = KMemoryBlockDisableMergeAttribute::None, | ||
| 1604 | }; | ||
| 1464 | } | 1605 | } |
| 1465 | 1606 | ||
| 1466 | return QueryInfoImpl(addr); | 1607 | return QueryInfoImpl(addr); |
| 1467 | } | 1608 | } |
| 1468 | 1609 | ||
| 1469 | Result KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { | ||
| 1470 | KScopedLightLock lk(general_lock); | ||
| 1471 | |||
| 1472 | KMemoryState state{}; | ||
| 1473 | KMemoryAttribute attribute{}; | ||
| 1474 | |||
| 1475 | R_TRY(CheckMemoryState(&state, nullptr, &attribute, nullptr, addr, size, | ||
| 1476 | KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, | ||
| 1477 | KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, | ||
| 1478 | KMemoryPermission::All, KMemoryPermission::UserReadWrite, | ||
| 1479 | KMemoryAttribute::Mask, KMemoryAttribute::None, | ||
| 1480 | KMemoryAttribute::IpcAndDeviceMapped)); | ||
| 1481 | |||
| 1482 | block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked); | ||
| 1483 | |||
| 1484 | return ResultSuccess; | ||
| 1485 | } | ||
| 1486 | |||
| 1487 | Result KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { | ||
| 1488 | KScopedLightLock lk(general_lock); | ||
| 1489 | |||
| 1490 | KMemoryState state{}; | ||
| 1491 | |||
| 1492 | R_TRY(CheckMemoryState(&state, nullptr, nullptr, nullptr, addr, size, | ||
| 1493 | KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, | ||
| 1494 | KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, | ||
| 1495 | KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask, | ||
| 1496 | KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); | ||
| 1497 | |||
| 1498 | block_manager->Update(addr, size / PageSize, state, KMemoryPermission::UserReadWrite); | ||
| 1499 | return ResultSuccess; | ||
| 1500 | } | ||
| 1501 | |||
| 1502 | Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, | 1610 | Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, |
| 1503 | Svc::MemoryPermission svc_perm) { | 1611 | Svc::MemoryPermission svc_perm) { |
| 1504 | const size_t num_pages = size / PageSize; | 1612 | const size_t num_pages = size / PageSize; |
| @@ -1509,20 +1617,30 @@ Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, | |||
| 1509 | // Verify we can change the memory permission. | 1617 | // Verify we can change the memory permission. |
| 1510 | KMemoryState old_state; | 1618 | KMemoryState old_state; |
| 1511 | KMemoryPermission old_perm; | 1619 | KMemoryPermission old_perm; |
| 1512 | R_TRY(this->CheckMemoryState( | 1620 | size_t num_allocator_blocks; |
| 1513 | std::addressof(old_state), std::addressof(old_perm), nullptr, nullptr, addr, size, | 1621 | R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), nullptr, |
| 1514 | KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, KMemoryPermission::None, | 1622 | std::addressof(num_allocator_blocks), addr, size, |
| 1515 | KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); | 1623 | KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, |
| 1624 | KMemoryPermission::None, KMemoryPermission::None, | ||
| 1625 | KMemoryAttribute::All, KMemoryAttribute::None)); | ||
| 1516 | 1626 | ||
| 1517 | // Determine new perm. | 1627 | // Determine new perm. |
| 1518 | const KMemoryPermission new_perm = ConvertToKMemoryPermission(svc_perm); | 1628 | const KMemoryPermission new_perm = ConvertToKMemoryPermission(svc_perm); |
| 1519 | R_SUCCEED_IF(old_perm == new_perm); | 1629 | R_SUCCEED_IF(old_perm == new_perm); |
| 1520 | 1630 | ||
| 1631 | // Create an update allocator. | ||
| 1632 | Result allocator_result{ResultSuccess}; | ||
| 1633 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1634 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1635 | R_TRY(allocator_result); | ||
| 1636 | |||
| 1521 | // Perform mapping operation. | 1637 | // Perform mapping operation. |
| 1522 | R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); | 1638 | R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); |
| 1523 | 1639 | ||
| 1524 | // Update the blocks. | 1640 | // Update the blocks. |
| 1525 | block_manager->Update(addr, num_pages, old_state, new_perm, KMemoryAttribute::None); | 1641 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, |
| 1642 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, | ||
| 1643 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1526 | 1644 | ||
| 1527 | return ResultSuccess; | 1645 | return ResultSuccess; |
| 1528 | } | 1646 | } |
| @@ -1548,6 +1666,12 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 | |||
| 1548 | KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, | 1666 | KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, |
| 1549 | AttributeTestMask, KMemoryAttribute::None, ~AttributeTestMask)); | 1667 | AttributeTestMask, KMemoryAttribute::None, ~AttributeTestMask)); |
| 1550 | 1668 | ||
| 1669 | // Create an update allocator. | ||
| 1670 | Result allocator_result{ResultSuccess}; | ||
| 1671 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1672 | memory_block_slab_manager, num_allocator_blocks); | ||
| 1673 | R_TRY(allocator_result); | ||
| 1674 | |||
| 1551 | // Determine the new attribute. | 1675 | // Determine the new attribute. |
| 1552 | const KMemoryAttribute new_attr = | 1676 | const KMemoryAttribute new_attr = |
| 1553 | static_cast<KMemoryAttribute>(((old_attr & static_cast<KMemoryAttribute>(~mask)) | | 1677 | static_cast<KMemoryAttribute>(((old_attr & static_cast<KMemoryAttribute>(~mask)) | |
| @@ -1557,7 +1681,9 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 | |||
| 1557 | this->Operate(addr, num_pages, old_perm, OperationType::ChangePermissionsAndRefresh); | 1681 | this->Operate(addr, num_pages, old_perm, OperationType::ChangePermissionsAndRefresh); |
| 1558 | 1682 | ||
| 1559 | // Update the blocks. | 1683 | // Update the blocks. |
| 1560 | block_manager->Update(addr, num_pages, old_state, old_perm, new_attr); | 1684 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, old_perm, |
| 1685 | new_attr, KMemoryBlockDisableMergeAttribute::None, | ||
| 1686 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1561 | 1687 | ||
| 1562 | return ResultSuccess; | 1688 | return ResultSuccess; |
| 1563 | } | 1689 | } |
| @@ -1603,6 +1729,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { | |||
| 1603 | KMemoryPermission::All, KMemoryPermission::UserReadWrite, | 1729 | KMemoryPermission::All, KMemoryPermission::UserReadWrite, |
| 1604 | KMemoryAttribute::All, KMemoryAttribute::None)); | 1730 | KMemoryAttribute::All, KMemoryAttribute::None)); |
| 1605 | 1731 | ||
| 1732 | // Create an update allocator. | ||
| 1733 | Result allocator_result{ResultSuccess}; | ||
| 1734 | KMemoryBlockManagerUpdateAllocator allocator( | ||
| 1735 | std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); | ||
| 1736 | R_TRY(allocator_result); | ||
| 1737 | |||
| 1606 | // Unmap the end of the heap. | 1738 | // Unmap the end of the heap. |
| 1607 | const auto num_pages = (GetHeapSize() - size) / PageSize; | 1739 | const auto num_pages = (GetHeapSize() - size) / PageSize; |
| 1608 | R_TRY(Operate(heap_region_start + size, num_pages, KMemoryPermission::None, | 1740 | R_TRY(Operate(heap_region_start + size, num_pages, KMemoryPermission::None, |
| @@ -1613,8 +1745,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { | |||
| 1613 | LimitableResource::PhysicalMemory, num_pages * PageSize); | 1745 | LimitableResource::PhysicalMemory, num_pages * PageSize); |
| 1614 | 1746 | ||
| 1615 | // Apply the memory block update. | 1747 | // Apply the memory block update. |
| 1616 | block_manager->Update(heap_region_start + size, num_pages, KMemoryState::Free, | 1748 | memory_block_manager.Update(std::addressof(allocator), heap_region_start + size, |
| 1617 | KMemoryPermission::None, KMemoryAttribute::None); | 1749 | num_pages, KMemoryState::Free, KMemoryPermission::None, |
| 1750 | KMemoryAttribute::None, | ||
| 1751 | KMemoryBlockDisableMergeAttribute::None, | ||
| 1752 | size == 0 ? KMemoryBlockDisableMergeAttribute::Normal | ||
| 1753 | : KMemoryBlockDisableMergeAttribute::None); | ||
| 1618 | 1754 | ||
| 1619 | // Update the current heap end. | 1755 | // Update the current heap end. |
| 1620 | current_heap_end = heap_region_start + size; | 1756 | current_heap_end = heap_region_start + size; |
| @@ -1667,6 +1803,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { | |||
| 1667 | KMemoryPermission::None, KMemoryPermission::None, | 1803 | KMemoryPermission::None, KMemoryPermission::None, |
| 1668 | KMemoryAttribute::None, KMemoryAttribute::None)); | 1804 | KMemoryAttribute::None, KMemoryAttribute::None)); |
| 1669 | 1805 | ||
| 1806 | // Create an update allocator. | ||
| 1807 | Result allocator_result{ResultSuccess}; | ||
| 1808 | KMemoryBlockManagerUpdateAllocator allocator( | ||
| 1809 | std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); | ||
| 1810 | R_TRY(allocator_result); | ||
| 1811 | |||
| 1670 | // Map the pages. | 1812 | // Map the pages. |
| 1671 | const auto num_pages = allocation_size / PageSize; | 1813 | const auto num_pages = allocation_size / PageSize; |
| 1672 | R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); | 1814 | R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); |
| @@ -1681,8 +1823,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { | |||
| 1681 | memory_reservation.Commit(); | 1823 | memory_reservation.Commit(); |
| 1682 | 1824 | ||
| 1683 | // Apply the memory block update. | 1825 | // Apply the memory block update. |
| 1684 | block_manager->Update(current_heap_end, num_pages, KMemoryState::Normal, | 1826 | memory_block_manager.Update( |
| 1685 | KMemoryPermission::UserReadWrite, KMemoryAttribute::None); | 1827 | std::addressof(allocator), current_heap_end, num_pages, KMemoryState::Normal, |
| 1828 | KMemoryPermission::UserReadWrite, KMemoryAttribute::None, | ||
| 1829 | heap_region_start == current_heap_end ? KMemoryBlockDisableMergeAttribute::Normal | ||
| 1830 | : KMemoryBlockDisableMergeAttribute::None, | ||
| 1831 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1686 | 1832 | ||
| 1687 | // Update the current heap end. | 1833 | // Update the current heap end. |
| 1688 | current_heap_end = heap_region_start + size; | 1834 | current_heap_end = heap_region_start + size; |
| @@ -1713,6 +1859,11 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, | |||
| 1713 | return ResultOutOfMemory; | 1859 | return ResultOutOfMemory; |
| 1714 | } | 1860 | } |
| 1715 | 1861 | ||
| 1862 | // Create an update allocator. | ||
| 1863 | Result allocator_result{ResultSuccess}; | ||
| 1864 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 1865 | memory_block_slab_manager); | ||
| 1866 | |||
| 1716 | if (is_map_only) { | 1867 | if (is_map_only) { |
| 1717 | R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); | 1868 | R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); |
| 1718 | } else { | 1869 | } else { |
| @@ -1723,53 +1874,38 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, | |||
| 1723 | R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); | 1874 | R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); |
| 1724 | } | 1875 | } |
| 1725 | 1876 | ||
| 1726 | block_manager->Update(addr, needed_num_pages, state, perm); | 1877 | // Update the blocks. |
| 1878 | memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, | ||
| 1879 | KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, | ||
| 1880 | KMemoryBlockDisableMergeAttribute::None); | ||
| 1727 | 1881 | ||
| 1728 | return addr; | 1882 | return addr; |
| 1729 | } | 1883 | } |
| 1730 | 1884 | ||
| 1731 | Result KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { | 1885 | Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, std::size_t size) { |
| 1732 | KScopedLightLock lk(general_lock); | 1886 | // Lightly validate the range before doing anything else. |
| 1733 | 1887 | const size_t num_pages = size / PageSize; | |
| 1734 | KMemoryPermission perm{}; | 1888 | R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); |
| 1735 | if (const Result result{CheckMemoryState( | ||
| 1736 | nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, | ||
| 1737 | KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, | ||
| 1738 | KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, | ||
| 1739 | KMemoryAttribute::DeviceSharedAndUncached)}; | ||
| 1740 | result.IsError()) { | ||
| 1741 | return result; | ||
| 1742 | } | ||
| 1743 | |||
| 1744 | block_manager->UpdateLock( | ||
| 1745 | addr, size / PageSize, | ||
| 1746 | [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { | ||
| 1747 | block->ShareToDevice(permission); | ||
| 1748 | }, | ||
| 1749 | perm); | ||
| 1750 | |||
| 1751 | return ResultSuccess; | ||
| 1752 | } | ||
| 1753 | 1889 | ||
| 1754 | Result KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { | 1890 | // Lock the table. |
| 1755 | KScopedLightLock lk(general_lock); | 1891 | KScopedLightLock lk(general_lock); |
| 1756 | 1892 | ||
| 1757 | KMemoryPermission perm{}; | 1893 | // Check the memory state. |
| 1758 | if (const Result result{CheckMemoryState( | 1894 | size_t num_allocator_blocks; |
| 1759 | nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, | 1895 | R_TRY(this->CheckMemoryStateContiguous( |
| 1760 | KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, | 1896 | std::addressof(num_allocator_blocks), address, size, KMemoryState::FlagCanDeviceMap, |
| 1761 | KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, | 1897 | KMemoryState::FlagCanDeviceMap, KMemoryPermission::None, KMemoryPermission::None, |
| 1762 | KMemoryAttribute::DeviceSharedAndUncached)}; | 1898 | KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); |
| 1763 | result.IsError()) { | ||
| 1764 | return result; | ||
| 1765 | } | ||
| 1766 | 1899 | ||
| 1767 | block_manager->UpdateLock( | 1900 | // Create an update allocator. |
| 1768 | addr, size / PageSize, | 1901 | Result allocator_result{ResultSuccess}; |
| 1769 | [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { | 1902 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), |
| 1770 | block->UnshareToDevice(permission); | 1903 | memory_block_slab_manager, num_allocator_blocks); |
| 1771 | }, | 1904 | R_TRY(allocator_result); |
| 1772 | perm); | 1905 | |
| 1906 | // Update the memory blocks. | ||
| 1907 | memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, | ||
| 1908 | &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); | ||
| 1773 | 1909 | ||
| 1774 | return ResultSuccess; | 1910 | return ResultSuccess; |
| 1775 | } | 1911 | } |
| @@ -1791,19 +1927,6 @@ Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPage | |||
| 1791 | KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); | 1927 | KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); |
| 1792 | } | 1928 | } |
| 1793 | 1929 | ||
| 1794 | Result KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { | ||
| 1795 | block_manager = std::make_unique<KMemoryBlockManager>(start, end); | ||
| 1796 | |||
| 1797 | return ResultSuccess; | ||
| 1798 | } | ||
| 1799 | |||
| 1800 | bool KPageTable::IsRegionMapped(VAddr address, u64 size) { | ||
| 1801 | return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, | ||
| 1802 | KMemoryPermission::All, KMemoryPermission::None, KMemoryAttribute::Mask, | ||
| 1803 | KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped) | ||
| 1804 | .IsError(); | ||
| 1805 | } | ||
| 1806 | |||
| 1807 | bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { | 1930 | bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { |
| 1808 | auto start_ptr = system.DeviceMemory().GetPointer<u8>(addr); | 1931 | auto start_ptr = system.DeviceMemory().GetPointer<u8>(addr); |
| 1809 | for (u64 offset{}; offset < size; offset += PageSize) { | 1932 | for (u64 offset{}; offset < size; offset += PageSize) { |
| @@ -1831,8 +1954,8 @@ VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_page | |||
| 1831 | if (is_aslr_enabled) { | 1954 | if (is_aslr_enabled) { |
| 1832 | UNIMPLEMENTED(); | 1955 | UNIMPLEMENTED(); |
| 1833 | } | 1956 | } |
| 1834 | return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, | 1957 | return memory_block_manager.FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, |
| 1835 | IsKernel() ? 1 : 4); | 1958 | IsKernel() ? 1 : 4); |
| 1836 | } | 1959 | } |
| 1837 | 1960 | ||
| 1838 | Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, | 1961 | Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, |
| @@ -2008,9 +2131,9 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ | |||
| 2008 | KMemoryPermission perm, KMemoryAttribute attr_mask, | 2131 | KMemoryPermission perm, KMemoryAttribute attr_mask, |
| 2009 | KMemoryAttribute attr) const { | 2132 | KMemoryAttribute attr) const { |
| 2010 | // Validate the states match expectation. | 2133 | // Validate the states match expectation. |
| 2011 | R_UNLESS((info.state & state_mask) == state, ResultInvalidCurrentMemory); | 2134 | R_UNLESS((info.m_state & state_mask) == state, ResultInvalidCurrentMemory); |
| 2012 | R_UNLESS((info.perm & perm_mask) == perm, ResultInvalidCurrentMemory); | 2135 | R_UNLESS((info.m_permission & perm_mask) == perm, ResultInvalidCurrentMemory); |
| 2013 | R_UNLESS((info.attribute & attr_mask) == attr, ResultInvalidCurrentMemory); | 2136 | R_UNLESS((info.m_attribute & attr_mask) == attr, ResultInvalidCurrentMemory); |
| 2014 | 2137 | ||
| 2015 | return ResultSuccess; | 2138 | return ResultSuccess; |
| 2016 | } | 2139 | } |
| @@ -2024,7 +2147,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA | |||
| 2024 | 2147 | ||
| 2025 | // Get information about the first block. | 2148 | // Get information about the first block. |
| 2026 | const VAddr last_addr = addr + size - 1; | 2149 | const VAddr last_addr = addr + size - 1; |
| 2027 | KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); | 2150 | KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); |
| 2028 | KMemoryInfo info = it->GetMemoryInfo(); | 2151 | KMemoryInfo info = it->GetMemoryInfo(); |
| 2029 | 2152 | ||
| 2030 | // If the start address isn't aligned, we need a block. | 2153 | // If the start address isn't aligned, we need a block. |
| @@ -2042,7 +2165,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA | |||
| 2042 | 2165 | ||
| 2043 | // Advance our iterator. | 2166 | // Advance our iterator. |
| 2044 | it++; | 2167 | it++; |
| 2045 | ASSERT(it != block_manager->cend()); | 2168 | ASSERT(it != memory_block_manager.cend()); |
| 2046 | info = it->GetMemoryInfo(); | 2169 | info = it->GetMemoryInfo(); |
| 2047 | } | 2170 | } |
| 2048 | 2171 | ||
| @@ -2067,7 +2190,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* | |||
| 2067 | 2190 | ||
| 2068 | // Get information about the first block. | 2191 | // Get information about the first block. |
| 2069 | const VAddr last_addr = addr + size - 1; | 2192 | const VAddr last_addr = addr + size - 1; |
| 2070 | KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); | 2193 | KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); |
| 2071 | KMemoryInfo info = it->GetMemoryInfo(); | 2194 | KMemoryInfo info = it->GetMemoryInfo(); |
| 2072 | 2195 | ||
| 2073 | // If the start address isn't aligned, we need a block. | 2196 | // If the start address isn't aligned, we need a block. |
| @@ -2075,14 +2198,14 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* | |||
| 2075 | (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; | 2198 | (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; |
| 2076 | 2199 | ||
| 2077 | // Validate all blocks in the range have correct state. | 2200 | // Validate all blocks in the range have correct state. |
| 2078 | const KMemoryState first_state = info.state; | 2201 | const KMemoryState first_state = info.m_state; |
| 2079 | const KMemoryPermission first_perm = info.perm; | 2202 | const KMemoryPermission first_perm = info.m_permission; |
| 2080 | const KMemoryAttribute first_attr = info.attribute; | 2203 | const KMemoryAttribute first_attr = info.m_attribute; |
| 2081 | while (true) { | 2204 | while (true) { |
| 2082 | // Validate the current block. | 2205 | // Validate the current block. |
| 2083 | R_UNLESS(info.state == first_state, ResultInvalidCurrentMemory); | 2206 | R_UNLESS(info.m_state == first_state, ResultInvalidCurrentMemory); |
| 2084 | R_UNLESS(info.perm == first_perm, ResultInvalidCurrentMemory); | 2207 | R_UNLESS(info.m_permission == first_perm, ResultInvalidCurrentMemory); |
| 2085 | R_UNLESS((info.attribute | ignore_attr) == (first_attr | ignore_attr), | 2208 | R_UNLESS((info.m_attribute | ignore_attr) == (first_attr | ignore_attr), |
| 2086 | ResultInvalidCurrentMemory); | 2209 | ResultInvalidCurrentMemory); |
| 2087 | 2210 | ||
| 2088 | // Validate against the provided masks. | 2211 | // Validate against the provided masks. |
| @@ -2095,7 +2218,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* | |||
| 2095 | 2218 | ||
| 2096 | // Advance our iterator. | 2219 | // Advance our iterator. |
| 2097 | it++; | 2220 | it++; |
| 2098 | ASSERT(it != block_manager->cend()); | 2221 | ASSERT(it != memory_block_manager.cend()); |
| 2099 | info = it->GetMemoryInfo(); | 2222 | info = it->GetMemoryInfo(); |
| 2100 | } | 2223 | } |
| 2101 | 2224 | ||
| @@ -2162,6 +2285,12 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr | |||
| 2162 | R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); | 2285 | R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); |
| 2163 | } | 2286 | } |
| 2164 | 2287 | ||
| 2288 | // Create an update allocator. | ||
| 2289 | Result allocator_result{ResultSuccess}; | ||
| 2290 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 2291 | memory_block_slab_manager, num_allocator_blocks); | ||
| 2292 | R_TRY(allocator_result); | ||
| 2293 | |||
| 2165 | // Decide on new perm and attr. | 2294 | // Decide on new perm and attr. |
| 2166 | new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; | 2295 | new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; |
| 2167 | KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); | 2296 | KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); |
| @@ -2172,7 +2301,9 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr | |||
| 2172 | } | 2301 | } |
| 2173 | 2302 | ||
| 2174 | // Apply the memory block updates. | 2303 | // Apply the memory block updates. |
| 2175 | block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); | 2304 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, |
| 2305 | new_attr, KMemoryBlockDisableMergeAttribute::Locked, | ||
| 2306 | KMemoryBlockDisableMergeAttribute::None); | ||
| 2176 | 2307 | ||
| 2177 | return ResultSuccess; | 2308 | return ResultSuccess; |
| 2178 | } | 2309 | } |
| @@ -2213,13 +2344,21 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask | |||
| 2213 | new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; | 2344 | new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; |
| 2214 | KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); | 2345 | KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); |
| 2215 | 2346 | ||
| 2347 | // Create an update allocator. | ||
| 2348 | Result allocator_result{ResultSuccess}; | ||
| 2349 | KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), | ||
| 2350 | memory_block_slab_manager, num_allocator_blocks); | ||
| 2351 | R_TRY(allocator_result); | ||
| 2352 | |||
| 2216 | // Update permission, if we need to. | 2353 | // Update permission, if we need to. |
| 2217 | if (new_perm != old_perm) { | 2354 | if (new_perm != old_perm) { |
| 2218 | R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); | 2355 | R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); |
| 2219 | } | 2356 | } |
| 2220 | 2357 | ||
| 2221 | // Apply the memory block updates. | 2358 | // Apply the memory block updates. |
| 2222 | block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); | 2359 | memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, |
| 2360 | new_attr, KMemoryBlockDisableMergeAttribute::None, | ||
| 2361 | KMemoryBlockDisableMergeAttribute::Locked); | ||
| 2223 | 2362 | ||
| 2224 | return ResultSuccess; | 2363 | return ResultSuccess; |
| 2225 | } | 2364 | } |
diff --git a/src/core/hle/kernel/k_page_table.h b/src/core/hle/kernel/k_page_table.h index 25774f232..fa11a0fe3 100644 --- a/src/core/hle/kernel/k_page_table.h +++ b/src/core/hle/kernel/k_page_table.h | |||
| @@ -9,8 +9,10 @@ | |||
| 9 | #include "common/common_types.h" | 9 | #include "common/common_types.h" |
| 10 | #include "common/page_table.h" | 10 | #include "common/page_table.h" |
| 11 | #include "core/file_sys/program_metadata.h" | 11 | #include "core/file_sys/program_metadata.h" |
| 12 | #include "core/hle/kernel/k_dynamic_resource_manager.h" | ||
| 12 | #include "core/hle/kernel/k_light_lock.h" | 13 | #include "core/hle/kernel/k_light_lock.h" |
| 13 | #include "core/hle/kernel/k_memory_block.h" | 14 | #include "core/hle/kernel/k_memory_block.h" |
| 15 | #include "core/hle/kernel/k_memory_block_manager.h" | ||
| 14 | #include "core/hle/kernel/k_memory_layout.h" | 16 | #include "core/hle/kernel/k_memory_layout.h" |
| 15 | #include "core/hle/kernel/k_memory_manager.h" | 17 | #include "core/hle/kernel/k_memory_manager.h" |
| 16 | #include "core/hle/result.h" | 18 | #include "core/hle/result.h" |
| @@ -34,7 +36,12 @@ public: | |||
| 34 | ~KPageTable(); | 36 | ~KPageTable(); |
| 35 | 37 | ||
| 36 | Result InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, | 38 | Result InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, |
| 37 | VAddr code_addr, std::size_t code_size, KMemoryManager::Pool pool); | 39 | VAddr code_addr, std::size_t code_size, |
| 40 | KMemoryBlockSlabManager* mem_block_slab_manager, | ||
| 41 | KMemoryManager::Pool pool); | ||
| 42 | |||
| 43 | void Finalize(); | ||
| 44 | |||
| 38 | Result MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, | 45 | Result MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, |
| 39 | KMemoryPermission perm); | 46 | KMemoryPermission perm); |
| 40 | Result MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size); | 47 | Result MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size); |
| @@ -58,8 +65,6 @@ public: | |||
| 58 | Result UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state); | 65 | Result UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state); |
| 59 | Result SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm); | 66 | Result SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm); |
| 60 | KMemoryInfo QueryInfo(VAddr addr); | 67 | KMemoryInfo QueryInfo(VAddr addr); |
| 61 | Result ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm); | ||
| 62 | Result ResetTransferMemory(VAddr addr, std::size_t size); | ||
| 63 | Result SetMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission perm); | 68 | Result SetMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission perm); |
| 64 | Result SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr); | 69 | Result SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr); |
| 65 | Result SetMaxHeapSize(std::size_t size); | 70 | Result SetMaxHeapSize(std::size_t size); |
| @@ -68,7 +73,6 @@ public: | |||
| 68 | bool is_map_only, VAddr region_start, | 73 | bool is_map_only, VAddr region_start, |
| 69 | std::size_t region_num_pages, KMemoryState state, | 74 | std::size_t region_num_pages, KMemoryState state, |
| 70 | KMemoryPermission perm, PAddr map_addr = 0); | 75 | KMemoryPermission perm, PAddr map_addr = 0); |
| 71 | Result LockForDeviceAddressSpace(VAddr addr, std::size_t size); | ||
| 72 | Result UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); | 76 | Result UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); |
| 73 | Result LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size); | 77 | Result LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size); |
| 74 | Result UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg); | 78 | Result UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg); |
| @@ -96,17 +100,14 @@ private: | |||
| 96 | ChangePermissionsAndRefresh, | 100 | ChangePermissionsAndRefresh, |
| 97 | }; | 101 | }; |
| 98 | 102 | ||
| 99 | static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = KMemoryAttribute::DontCareMask | | 103 | static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = |
| 100 | KMemoryAttribute::IpcLocked | | 104 | KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared; |
| 101 | KMemoryAttribute::DeviceShared; | ||
| 102 | 105 | ||
| 103 | Result InitializeMemoryLayout(VAddr start, VAddr end); | ||
| 104 | Result MapPages(VAddr addr, const KPageGroup& page_linked_list, KMemoryPermission perm); | 106 | Result MapPages(VAddr addr, const KPageGroup& page_linked_list, KMemoryPermission perm); |
| 105 | Result MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, PAddr phys_addr, | 107 | Result MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, PAddr phys_addr, |
| 106 | bool is_pa_valid, VAddr region_start, std::size_t region_num_pages, | 108 | bool is_pa_valid, VAddr region_start, std::size_t region_num_pages, |
| 107 | KMemoryState state, KMemoryPermission perm); | 109 | KMemoryState state, KMemoryPermission perm); |
| 108 | Result UnmapPages(VAddr addr, const KPageGroup& page_linked_list); | 110 | Result UnmapPages(VAddr addr, const KPageGroup& page_linked_list); |
| 109 | bool IsRegionMapped(VAddr address, u64 size); | ||
| 110 | bool IsRegionContiguous(VAddr addr, u64 size) const; | 111 | bool IsRegionContiguous(VAddr addr, u64 size) const; |
| 111 | void AddRegionToPages(VAddr start, std::size_t num_pages, KPageGroup& page_linked_list); | 112 | void AddRegionToPages(VAddr start, std::size_t num_pages, KPageGroup& page_linked_list); |
| 112 | KMemoryInfo QueryInfoImpl(VAddr addr); | 113 | KMemoryInfo QueryInfoImpl(VAddr addr); |
| @@ -194,8 +195,6 @@ private: | |||
| 194 | mutable KLightLock general_lock; | 195 | mutable KLightLock general_lock; |
| 195 | mutable KLightLock map_physical_memory_lock; | 196 | mutable KLightLock map_physical_memory_lock; |
| 196 | 197 | ||
| 197 | std::unique_ptr<KMemoryBlockManager> block_manager; | ||
| 198 | |||
| 199 | public: | 198 | public: |
| 200 | constexpr VAddr GetAddressSpaceStart() const { | 199 | constexpr VAddr GetAddressSpaceStart() const { |
| 201 | return address_space_start; | 200 | return address_space_start; |
| @@ -346,9 +345,13 @@ private: | |||
| 346 | std::size_t max_physical_memory_size{}; | 345 | std::size_t max_physical_memory_size{}; |
| 347 | std::size_t address_space_width{}; | 346 | std::size_t address_space_width{}; |
| 348 | 347 | ||
| 348 | KMemoryBlockManager memory_block_manager; | ||
| 349 | |||
| 349 | bool is_kernel{}; | 350 | bool is_kernel{}; |
| 350 | bool is_aslr_enabled{}; | 351 | bool is_aslr_enabled{}; |
| 351 | 352 | ||
| 353 | KMemoryBlockSlabManager* memory_block_slab_manager{}; | ||
| 354 | |||
| 352 | u32 heap_fill_value{}; | 355 | u32 heap_fill_value{}; |
| 353 | const KMemoryRegion* cached_physical_heap_region{}; | 356 | const KMemoryRegion* cached_physical_heap_region{}; |
| 354 | 357 | ||