summaryrefslogtreecommitdiff
path: root/src/core/memory.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/core/memory.cpp')
-rw-r--r--src/core/memory.cpp354
1 files changed, 211 insertions, 143 deletions
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index ebe16b030..462d68386 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -4,95 +4,53 @@
4 4
5#include <array> 5#include <array>
6#include <cstring> 6#include <cstring>
7#include "audio_core/audio_core.h"
7#include "common/assert.h" 8#include "common/assert.h"
8#include "common/common_types.h" 9#include "common/common_types.h"
9#include "common/logging/log.h" 10#include "common/logging/log.h"
10#include "common/swap.h" 11#include "common/swap.h"
12#include "core/arm/arm_interface.h"
13#include "core/core.h"
14#include "core/hle/kernel/memory.h"
11#include "core/hle/kernel/process.h" 15#include "core/hle/kernel/process.h"
16#include "core/hle/lock.h"
12#include "core/memory.h" 17#include "core/memory.h"
13#include "core/memory_setup.h" 18#include "core/memory_setup.h"
14#include "core/mmio.h"
15#include "video_core/renderer_base.h" 19#include "video_core/renderer_base.h"
16#include "video_core/video_core.h" 20#include "video_core/video_core.h"
17 21
18namespace Memory { 22namespace Memory {
19 23
20enum class PageType { 24static std::array<u8, Memory::VRAM_SIZE> vram;
21 /// Page is unmapped and should cause an access error. 25static std::array<u8, Memory::N3DS_EXTRA_RAM_SIZE> n3ds_extra_ram;
22 Unmapped,
23 /// Page is mapped to regular memory. This is the only type you can get pointers to.
24 Memory,
25 /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
26 /// invalidation
27 RasterizerCachedMemory,
28 /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions.
29 Special,
30 /// Page is mapped to a I/O region, but also needs to check for rasterizer cache flushing and
31 /// invalidation
32 RasterizerCachedSpecial,
33};
34
35struct SpecialRegion {
36 VAddr base;
37 u64 size;
38 MMIORegionPointer handler;
39};
40 26
41/** 27static PageTable* current_page_table = nullptr;
42 * A (reasonably) fast way of allowing switchable and remappable process address spaces. It loosely 28
43 * mimics the way a real CPU page table works, but instead is optimized for minimal decoding and 29void SetCurrentPageTable(PageTable* page_table) {
44 * fetching requirements when accessing. In the usual case of an access to regular memory, it only 30 current_page_table = page_table;
45 * requires an indexed fetch and a check for NULL. 31 if (Core::System::GetInstance().IsPoweredOn()) {
46 */ 32 Core::CPU().PageTableChanged();
47struct PageTable { 33 }
48 /** 34}
49 * Array of memory pointers backing each page. An entry can only be non-null if the 35
50 * corresponding entry in the `attributes` array is of type `Memory`. 36PageTable* GetCurrentPageTable() {
51 */ 37 return current_page_table;
52 std::map<u64, u8*> pointers; 38}
53 39
54 /** 40static void MapPages(PageTable& page_table, VAddr base, u32 size, u8* memory, PageType type) {
55 * Contains MMIO handlers that back memory regions whose entries in the `attribute` array is of
56 * type `Special`.
57 */
58 std::vector<SpecialRegion> special_regions;
59
60 /**
61 * Array of fine grained page attributes. If it is set to any value other than `Memory`, then
62 * the corresponding entry in `pointers` MUST be set to null.
63 */
64 std::map<u64, PageType> attributes;
65
66 /**
67 * Indicates the number of externally cached resources touching a page that should be
68 * flushed before the memory is accessed
69 */
70 std::map<u64, u8> cached_res_count;
71};
72
73/// Singular page table used for the singleton process
74static PageTable main_page_table;
75/// Currently active page table
76static PageTable* current_page_table = &main_page_table;
77
78//std::array<u8*, PAGE_TABLE_NUM_ENTRIES>* GetCurrentPageTablePointers() {
79// return &current_page_table->pointers;
80//}
81
82static void MapPages(u64 base, u64 size, u8* memory, PageType type) {
83 LOG_DEBUG(HW_Memory, "Mapping %p onto %08X-%08X", memory, base * PAGE_SIZE, 41 LOG_DEBUG(HW_Memory, "Mapping %p onto %08X-%08X", memory, base * PAGE_SIZE,
84 (base + size) * PAGE_SIZE); 42 (base + size) * PAGE_SIZE);
85 43
86 RasterizerFlushVirtualRegion(base << PAGE_BITS, size * PAGE_SIZE, 44 RasterizerFlushVirtualRegion(base << PAGE_BITS, size * PAGE_SIZE,
87 FlushMode::FlushAndInvalidate); 45 FlushMode::FlushAndInvalidate);
88 46
89 u64 end = base + size; 47 VAddr end = base + size;
90 while (base != end) { 48 while (base != end) {
91 ASSERT_MSG(base < PAGE_TABLE_NUM_ENTRIES, "out of range mapping at %08X", base); 49 ASSERT_MSG(base < PAGE_TABLE_NUM_ENTRIES, "out of range mapping at %08X", base);
92 50
93 current_page_table->attributes[base] = type; 51 page_table.attributes[base] = type;
94 current_page_table->pointers[base] = memory; 52 page_table.pointers[base] = memory;
95 current_page_table->cached_res_count[base] = 0; 53 page_table.cached_res_count[base] = 0;
96 54
97 base += 1; 55 base += 1;
98 if (memory != nullptr) 56 if (memory != nullptr)
@@ -100,40 +58,34 @@ static void MapPages(u64 base, u64 size, u8* memory, PageType type) {
100 } 58 }
101} 59}
102 60
103void InitMemoryMap() { 61void MapMemoryRegion(PageTable& page_table, VAddr base, u32 size, u8* target) {
104 //main_page_table.pointers.fill(nullptr);
105 //main_page_table.attributes.fill(PageType::Unmapped);
106 //main_page_table.cached_res_count.fill(0);
107}
108
109void MapMemoryRegion(VAddr base, u64 size, u8* target) {
110 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size); 62 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size);
111 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base); 63 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base);
112 MapPages(base / PAGE_SIZE, size / PAGE_SIZE, target, PageType::Memory); 64 MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, PageType::Memory);
113} 65}
114 66
115void MapIoRegion(VAddr base, u64 size, MMIORegionPointer mmio_handler) { 67void MapIoRegion(PageTable& page_table, VAddr base, u32 size, MMIORegionPointer mmio_handler) {
116 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size); 68 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size);
117 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base); 69 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base);
118 MapPages(base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Special); 70 MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Special);
119 71
120 current_page_table->special_regions.emplace_back(SpecialRegion{base, size, mmio_handler}); 72 page_table.special_regions.emplace_back(SpecialRegion{base, size, mmio_handler});
121} 73}
122 74
123void UnmapRegion(VAddr base, u64 size) { 75void UnmapRegion(PageTable& page_table, VAddr base, u32 size) {
124 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size); 76 ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size);
125 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base); 77 ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base);
126 MapPages(base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Unmapped); 78 MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Unmapped);
127} 79}
128 80
129/** 81/**
130 * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) 82 * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
131 * using a VMA from the current process 83 * using a VMA from the current process
132 */ 84 */
133static u8* GetPointerFromVMA(VAddr vaddr) { 85static u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) {
134 u8* direct_pointer = nullptr; 86 u8* direct_pointer = nullptr;
135 87
136 auto& vm_manager = Kernel::g_current_process->vm_manager; 88 auto& vm_manager = process.vm_manager;
137 89
138 auto it = vm_manager.FindVMA(vaddr); 90 auto it = vm_manager.FindVMA(vaddr);
139 ASSERT(it != vm_manager.vma_map.end()); 91 ASSERT(it != vm_manager.vma_map.end());
@@ -156,10 +108,18 @@ static u8* GetPointerFromVMA(VAddr vaddr) {
156} 108}
157 109
158/** 110/**
111 * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
112 * using a VMA from the current process.
113 */
114static u8* GetPointerFromVMA(VAddr vaddr) {
115 return GetPointerFromVMA(*Kernel::g_current_process, vaddr);
116}
117
118/**
159 * This function should only be called for virtual addreses with attribute `PageType::Special`. 119 * This function should only be called for virtual addreses with attribute `PageType::Special`.
160 */ 120 */
161static MMIORegionPointer GetMMIOHandler(VAddr vaddr) { 121static MMIORegionPointer GetMMIOHandler(const PageTable& page_table, VAddr vaddr) {
162 for (const auto& region : current_page_table->special_regions) { 122 for (const auto& region : page_table.special_regions) {
163 if (vaddr >= region.base && vaddr < (region.base + region.size)) { 123 if (vaddr >= region.base && vaddr < (region.base + region.size)) {
164 return region.handler; 124 return region.handler;
165 } 125 }
@@ -168,6 +128,11 @@ static MMIORegionPointer GetMMIOHandler(VAddr vaddr) {
168 return nullptr; // Should never happen 128 return nullptr; // Should never happen
169} 129}
170 130
131static MMIORegionPointer GetMMIOHandler(VAddr vaddr) {
132 const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table;
133 return GetMMIOHandler(page_table, vaddr);
134}
135
171template <typename T> 136template <typename T>
172T ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr); 137T ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr);
173 138
@@ -181,10 +146,13 @@ T Read(const VAddr vaddr) {
181 return value; 146 return value;
182 } 147 }
183 148
149 // The memory access might do an MMIO or cached access, so we have to lock the HLE kernel state
150 std::lock_guard<std::recursive_mutex> lock(HLE::g_hle_lock);
151
184 PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; 152 PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
185 switch (type) { 153 switch (type) {
186 case PageType::Unmapped: 154 case PageType::Unmapped:
187 LOG_ERROR(HW_Memory, "unmapped Read%lu @ 0x%llx", sizeof(T) * 8, vaddr); 155 LOG_ERROR(HW_Memory, "unmapped Read%lu @ 0x%08X", sizeof(T) * 8, vaddr);
188 return 0; 156 return 0;
189 case PageType::Memory: 157 case PageType::Memory:
190 ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr); 158 ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr);
@@ -219,10 +187,13 @@ void Write(const VAddr vaddr, const T data) {
219 return; 187 return;
220 } 188 }
221 189
190 // The memory access might do an MMIO or cached access, so we have to lock the HLE kernel state
191 std::lock_guard<std::recursive_mutex> lock(HLE::g_hle_lock);
192
222 PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; 193 PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
223 switch (type) { 194 switch (type) {
224 case PageType::Unmapped: 195 case PageType::Unmapped:
225 LOG_ERROR(HW_Memory, "unmapped Write%lu 0x%llx @ 0x%llx", sizeof(data) * 8, (u64)data, 196 LOG_ERROR(HW_Memory, "unmapped Write%lu 0x%08X @ 0x%08X", sizeof(data) * 8, (u32)data,
226 vaddr); 197 vaddr);
227 return; 198 return;
228 case PageType::Memory: 199 case PageType::Memory:
@@ -246,18 +217,20 @@ void Write(const VAddr vaddr, const T data) {
246 } 217 }
247} 218}
248 219
249bool IsValidVirtualAddress(const VAddr vaddr) { 220bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) {
250 const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; 221 auto& page_table = process.vm_manager.page_table;
222
223 const u8* page_pointer = page_table.pointers[vaddr >> PAGE_BITS];
251 if (page_pointer) 224 if (page_pointer)
252 return true; 225 return true;
253 226
254 if (current_page_table->attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory) 227 if (page_table.attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory)
255 return true; 228 return true;
256 229
257 if (current_page_table->attributes[vaddr >> PAGE_BITS] != PageType::Special) 230 if (page_table.attributes[vaddr >> PAGE_BITS] != PageType::Special)
258 return false; 231 return false;
259 232
260 MMIORegionPointer mmio_region = GetMMIOHandler(vaddr); 233 MMIORegionPointer mmio_region = GetMMIOHandler(page_table, vaddr);
261 if (mmio_region) { 234 if (mmio_region) {
262 return mmio_region->IsValidAddress(vaddr); 235 return mmio_region->IsValidAddress(vaddr);
263 } 236 }
@@ -265,9 +238,12 @@ bool IsValidVirtualAddress(const VAddr vaddr) {
265 return false; 238 return false;
266} 239}
267 240
241bool IsValidVirtualAddress(const VAddr vaddr) {
242 return IsValidVirtualAddress(*Kernel::g_current_process, vaddr);
243}
244
268bool IsValidPhysicalAddress(const PAddr paddr) { 245bool IsValidPhysicalAddress(const PAddr paddr) {
269 boost::optional<VAddr> vaddr = PhysicalToVirtualAddress(paddr); 246 return GetPhysicalPointer(paddr) != nullptr;
270 return vaddr && IsValidVirtualAddress(*vaddr);
271} 247}
272 248
273u8* GetPointer(const VAddr vaddr) { 249u8* GetPointer(const VAddr vaddr) {
@@ -280,7 +256,7 @@ u8* GetPointer(const VAddr vaddr) {
280 return GetPointerFromVMA(vaddr); 256 return GetPointerFromVMA(vaddr);
281 } 257 }
282 258
283 LOG_ERROR(HW_Memory, "unknown GetPointer @ 0x%llx", vaddr); 259 LOG_ERROR(HW_Memory, "unknown GetPointer @ 0x%08x", vaddr);
284 return nullptr; 260 return nullptr;
285} 261}
286 262
@@ -299,12 +275,66 @@ std::string ReadCString(VAddr vaddr, std::size_t max_length) {
299} 275}
300 276
301u8* GetPhysicalPointer(PAddr address) { 277u8* GetPhysicalPointer(PAddr address) {
302 // TODO(Subv): This call should not go through the application's memory mapping. 278 struct MemoryArea {
303 boost::optional<VAddr> vaddr = PhysicalToVirtualAddress(address); 279 PAddr paddr_base;
304 return vaddr ? GetPointer(*vaddr) : nullptr; 280 u32 size;
281 };
282
283 static constexpr MemoryArea memory_areas[] = {
284 {VRAM_PADDR, VRAM_SIZE},
285 {IO_AREA_PADDR, IO_AREA_SIZE},
286 {DSP_RAM_PADDR, DSP_RAM_SIZE},
287 {FCRAM_PADDR, FCRAM_N3DS_SIZE},
288 {N3DS_EXTRA_RAM_PADDR, N3DS_EXTRA_RAM_SIZE},
289 };
290
291 const auto area =
292 std::find_if(std::begin(memory_areas), std::end(memory_areas), [&](const auto& area) {
293 return address >= area.paddr_base && address < area.paddr_base + area.size;
294 });
295
296 if (area == std::end(memory_areas)) {
297 LOG_ERROR(HW_Memory, "unknown GetPhysicalPointer @ 0x%08X", address);
298 return nullptr;
299 }
300
301 if (area->paddr_base == IO_AREA_PADDR) {
302 LOG_ERROR(HW_Memory, "MMIO mappings are not supported yet. phys_addr=0x%08X", address);
303 return nullptr;
304 }
305
306 u64 offset_into_region = address - area->paddr_base;
307
308 u8* target_pointer = nullptr;
309 switch (area->paddr_base) {
310 case VRAM_PADDR:
311 target_pointer = vram.data() + offset_into_region;
312 break;
313 case DSP_RAM_PADDR:
314 target_pointer = AudioCore::GetDspMemory().data() + offset_into_region;
315 break;
316 case FCRAM_PADDR:
317 for (const auto& region : Kernel::memory_regions) {
318 if (offset_into_region >= region.base &&
319 offset_into_region < region.base + region.size) {
320 target_pointer =
321 region.linear_heap_memory->data() + offset_into_region - region.base;
322 break;
323 }
324 }
325 ASSERT_MSG(target_pointer != nullptr, "Invalid FCRAM address");
326 break;
327 case N3DS_EXTRA_RAM_PADDR:
328 target_pointer = n3ds_extra_ram.data() + offset_into_region;
329 break;
330 default:
331 UNREACHABLE();
332 }
333
334 return target_pointer;
305} 335}
306 336
307void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) { 337void RasterizerMarkRegionCached(PAddr start, u32 size, int count_delta) {
308 if (start == 0) { 338 if (start == 0) {
309 return; 339 return;
310 } 340 }
@@ -314,8 +344,15 @@ void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) {
314 344
315 for (unsigned i = 0; i < num_pages; ++i, paddr += PAGE_SIZE) { 345 for (unsigned i = 0; i < num_pages; ++i, paddr += PAGE_SIZE) {
316 boost::optional<VAddr> maybe_vaddr = PhysicalToVirtualAddress(paddr); 346 boost::optional<VAddr> maybe_vaddr = PhysicalToVirtualAddress(paddr);
317 if (!maybe_vaddr) 347 // While the physical <-> virtual mapping is 1:1 for the regions supported by the cache,
348 // some games (like Pokemon Super Mystery Dungeon) will try to use textures that go beyond
349 // the end address of VRAM, causing the Virtual->Physical translation to fail when flushing
350 // parts of the texture.
351 if (!maybe_vaddr) {
352 LOG_ERROR(HW_Memory,
353 "Trying to flush a cached region to an invalid physical address %08X", paddr);
318 continue; 354 continue;
355 }
319 VAddr vaddr = *maybe_vaddr; 356 VAddr vaddr = *maybe_vaddr;
320 357
321 u8& res_count = current_page_table->cached_res_count[vaddr >> PAGE_BITS]; 358 u8& res_count = current_page_table->cached_res_count[vaddr >> PAGE_BITS];
@@ -327,6 +364,10 @@ void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) {
327 if (res_count == 0) { 364 if (res_count == 0) {
328 PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; 365 PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS];
329 switch (page_type) { 366 switch (page_type) {
367 case PageType::Unmapped:
368 // It is not necessary for a process to have this region mapped into its address
369 // space, for example, a system module need not have a VRAM mapping.
370 break;
330 case PageType::Memory: 371 case PageType::Memory:
331 page_type = PageType::RasterizerCachedMemory; 372 page_type = PageType::RasterizerCachedMemory;
332 current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; 373 current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr;
@@ -345,6 +386,10 @@ void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) {
345 if (res_count == 0) { 386 if (res_count == 0) {
346 PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; 387 PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS];
347 switch (page_type) { 388 switch (page_type) {
389 case PageType::Unmapped:
390 // It is not necessary for a process to have this region mapped into its address
391 // space, for example, a system module need not have a VRAM mapping.
392 break;
348 case PageType::RasterizerCachedMemory: { 393 case PageType::RasterizerCachedMemory: {
349 u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK); 394 u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK);
350 if (pointer == nullptr) { 395 if (pointer == nullptr) {
@@ -368,13 +413,13 @@ void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) {
368 } 413 }
369} 414}
370 415
371void RasterizerFlushRegion(PAddr start, u64 size) { 416void RasterizerFlushRegion(PAddr start, u32 size) {
372 if (VideoCore::g_renderer != nullptr) { 417 if (VideoCore::g_renderer != nullptr) {
373 VideoCore::g_renderer->Rasterizer()->FlushRegion(start, size); 418 VideoCore::g_renderer->Rasterizer()->FlushRegion(start, size);
374 } 419 }
375} 420}
376 421
377void RasterizerFlushAndInvalidateRegion(PAddr start, u64 size) { 422void RasterizerFlushAndInvalidateRegion(PAddr start, u32 size) {
378 // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be 423 // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be
379 // null here 424 // null here
380 if (VideoCore::g_renderer != nullptr) { 425 if (VideoCore::g_renderer != nullptr) {
@@ -382,7 +427,7 @@ void RasterizerFlushAndInvalidateRegion(PAddr start, u64 size) {
382 } 427 }
383} 428}
384 429
385void RasterizerFlushVirtualRegion(VAddr start, u64 size, FlushMode mode) { 430void RasterizerFlushVirtualRegion(VAddr start, u32 size, FlushMode mode) {
386 // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be 431 // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be
387 // null here 432 // null here
388 if (VideoCore::g_renderer != nullptr) { 433 if (VideoCore::g_renderer != nullptr) {
@@ -398,7 +443,7 @@ void RasterizerFlushVirtualRegion(VAddr start, u64 size, FlushMode mode) {
398 VAddr overlap_end = std::min(end, region_end); 443 VAddr overlap_end = std::min(end, region_end);
399 444
400 PAddr physical_start = TryVirtualToPhysicalAddress(overlap_start).value(); 445 PAddr physical_start = TryVirtualToPhysicalAddress(overlap_start).value();
401 u64 overlap_size = overlap_end - overlap_start; 446 u32 overlap_size = static_cast<u32>(overlap_end - overlap_start);
402 447
403 auto* rasterizer = VideoCore::g_renderer->Rasterizer(); 448 auto* rasterizer = VideoCore::g_renderer->Rasterizer();
404 switch (mode) { 449 switch (mode) {
@@ -433,44 +478,50 @@ u64 Read64(const VAddr addr) {
433 return Read<u64_le>(addr); 478 return Read<u64_le>(addr);
434} 479}
435 480
436void ReadBlock(const VAddr src_addr, void* dest_buffer, const size_t size) { 481void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer,
482 const size_t size) {
483 auto& page_table = process.vm_manager.page_table;
484
437 size_t remaining_size = size; 485 size_t remaining_size = size;
438 size_t page_index = src_addr >> PAGE_BITS; 486 size_t page_index = src_addr >> PAGE_BITS;
439 size_t page_offset = src_addr & PAGE_MASK; 487 size_t page_offset = src_addr & PAGE_MASK;
440 488
441 while (remaining_size > 0) { 489 while (remaining_size > 0) {
442 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size); 490 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size);
443 const VAddr current_vaddr = (page_index << PAGE_BITS) + page_offset; 491 const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
444 492
445 switch (current_page_table->attributes[page_index]) { 493 switch (page_table.attributes[page_index]) {
446 case PageType::Unmapped: { 494 case PageType::Unmapped: {
447 LOG_ERROR(HW_Memory, "unmapped ReadBlock @ 0x%llx (start address = 0x%llx, size = %zu)", 495 LOG_ERROR(HW_Memory, "unmapped ReadBlock @ 0x%08X (start address = 0x%08X, size = %zu)",
448 current_vaddr, src_addr, size); 496 current_vaddr, src_addr, size);
449 std::memset(dest_buffer, 0, copy_amount); 497 std::memset(dest_buffer, 0, copy_amount);
450 break; 498 break;
451 } 499 }
452 case PageType::Memory: { 500 case PageType::Memory: {
453 DEBUG_ASSERT(current_page_table->pointers[page_index]); 501 DEBUG_ASSERT(page_table.pointers[page_index]);
454 502
455 const u8* src_ptr = current_page_table->pointers[page_index] + page_offset; 503 const u8* src_ptr = page_table.pointers[page_index] + page_offset;
456 std::memcpy(dest_buffer, src_ptr, copy_amount); 504 std::memcpy(dest_buffer, src_ptr, copy_amount);
457 break; 505 break;
458 } 506 }
459 case PageType::Special: { 507 case PageType::Special: {
460 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 508 MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr);
461 509 DEBUG_ASSERT(handler);
462 GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, dest_buffer, copy_amount); 510 handler->ReadBlock(current_vaddr, dest_buffer, copy_amount);
463 break; 511 break;
464 } 512 }
465 case PageType::RasterizerCachedMemory: { 513 case PageType::RasterizerCachedMemory: {
466 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::Flush); 514 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
467 std::memcpy(dest_buffer, GetPointerFromVMA(current_vaddr), copy_amount); 515 FlushMode::Flush);
516 std::memcpy(dest_buffer, GetPointerFromVMA(process, current_vaddr), copy_amount);
468 break; 517 break;
469 } 518 }
470 case PageType::RasterizerCachedSpecial: { 519 case PageType::RasterizerCachedSpecial: {
471 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 520 MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr);
472 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::Flush); 521 DEBUG_ASSERT(handler);
473 GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, dest_buffer, copy_amount); 522 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
523 FlushMode::Flush);
524 handler->ReadBlock(current_vaddr, dest_buffer, copy_amount);
474 break; 525 break;
475 } 526 }
476 default: 527 default:
@@ -484,6 +535,10 @@ void ReadBlock(const VAddr src_addr, void* dest_buffer, const size_t size) {
484 } 535 }
485} 536}
486 537
538void ReadBlock(const VAddr src_addr, void* dest_buffer, const size_t size) {
539 ReadBlock(*Kernel::g_current_process, src_addr, dest_buffer, size);
540}
541
487void Write8(const VAddr addr, const u8 data) { 542void Write8(const VAddr addr, const u8 data) {
488 Write<u8>(addr, data); 543 Write<u8>(addr, data);
489} 544}
@@ -500,44 +555,49 @@ void Write64(const VAddr addr, const u64 data) {
500 Write<u64_le>(addr, data); 555 Write<u64_le>(addr, data);
501} 556}
502 557
503void WriteBlock(const VAddr dest_addr, const void* src_buffer, const size_t size) { 558void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer,
559 const size_t size) {
560 auto& page_table = process.vm_manager.page_table;
504 size_t remaining_size = size; 561 size_t remaining_size = size;
505 size_t page_index = dest_addr >> PAGE_BITS; 562 size_t page_index = dest_addr >> PAGE_BITS;
506 size_t page_offset = dest_addr & PAGE_MASK; 563 size_t page_offset = dest_addr & PAGE_MASK;
507 564
508 while (remaining_size > 0) { 565 while (remaining_size > 0) {
509 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size); 566 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size);
510 const VAddr current_vaddr = (page_index << PAGE_BITS) + page_offset; 567 const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
511 568
512 switch (current_page_table->attributes[page_index]) { 569 switch (page_table.attributes[page_index]) {
513 case PageType::Unmapped: { 570 case PageType::Unmapped: {
514 LOG_ERROR(HW_Memory, 571 LOG_ERROR(HW_Memory,
515 "unmapped WriteBlock @ 0x%llx (start address = 0x%llx, size = %zu)", 572 "unmapped WriteBlock @ 0x%08X (start address = 0x%08X, size = %zu)",
516 current_vaddr, dest_addr, size); 573 current_vaddr, dest_addr, size);
517 break; 574 break;
518 } 575 }
519 case PageType::Memory: { 576 case PageType::Memory: {
520 DEBUG_ASSERT(current_page_table->pointers[page_index]); 577 DEBUG_ASSERT(page_table.pointers[page_index]);
521 578
522 u8* dest_ptr = current_page_table->pointers[page_index] + page_offset; 579 u8* dest_ptr = page_table.pointers[page_index] + page_offset;
523 std::memcpy(dest_ptr, src_buffer, copy_amount); 580 std::memcpy(dest_ptr, src_buffer, copy_amount);
524 break; 581 break;
525 } 582 }
526 case PageType::Special: { 583 case PageType::Special: {
527 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 584 MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr);
528 585 DEBUG_ASSERT(handler);
529 GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, src_buffer, copy_amount); 586 handler->WriteBlock(current_vaddr, src_buffer, copy_amount);
530 break; 587 break;
531 } 588 }
532 case PageType::RasterizerCachedMemory: { 589 case PageType::RasterizerCachedMemory: {
533 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::FlushAndInvalidate); 590 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
534 std::memcpy(GetPointerFromVMA(current_vaddr), src_buffer, copy_amount); 591 FlushMode::FlushAndInvalidate);
592 std::memcpy(GetPointerFromVMA(process, current_vaddr), src_buffer, copy_amount);
535 break; 593 break;
536 } 594 }
537 case PageType::RasterizerCachedSpecial: { 595 case PageType::RasterizerCachedSpecial: {
538 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 596 MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr);
539 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::FlushAndInvalidate); 597 DEBUG_ASSERT(handler);
540 GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, src_buffer, copy_amount); 598 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
599 FlushMode::FlushAndInvalidate);
600 handler->WriteBlock(current_vaddr, src_buffer, copy_amount);
541 break; 601 break;
542 } 602 }
543 default: 603 default:
@@ -551,6 +611,10 @@ void WriteBlock(const VAddr dest_addr, const void* src_buffer, const size_t size
551 } 611 }
552} 612}
553 613
614void WriteBlock(const VAddr dest_addr, const void* src_buffer, const size_t size) {
615 WriteBlock(*Kernel::g_current_process, dest_addr, src_buffer, size);
616}
617
554void ZeroBlock(const VAddr dest_addr, const size_t size) { 618void ZeroBlock(const VAddr dest_addr, const size_t size) {
555 size_t remaining_size = size; 619 size_t remaining_size = size;
556 size_t page_index = dest_addr >> PAGE_BITS; 620 size_t page_index = dest_addr >> PAGE_BITS;
@@ -560,11 +624,11 @@ void ZeroBlock(const VAddr dest_addr, const size_t size) {
560 624
561 while (remaining_size > 0) { 625 while (remaining_size > 0) {
562 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size); 626 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size);
563 const VAddr current_vaddr = (page_index << PAGE_BITS) + page_offset; 627 const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
564 628
565 switch (current_page_table->attributes[page_index]) { 629 switch (current_page_table->attributes[page_index]) {
566 case PageType::Unmapped: { 630 case PageType::Unmapped: {
567 LOG_ERROR(HW_Memory, "unmapped ZeroBlock @ 0x%llx (start address = 0x%llx, size = %zu)", 631 LOG_ERROR(HW_Memory, "unmapped ZeroBlock @ 0x%08X (start address = 0x%08X, size = %zu)",
568 current_vaddr, dest_addr, size); 632 current_vaddr, dest_addr, size);
569 break; 633 break;
570 } 634 }
@@ -582,13 +646,15 @@ void ZeroBlock(const VAddr dest_addr, const size_t size) {
582 break; 646 break;
583 } 647 }
584 case PageType::RasterizerCachedMemory: { 648 case PageType::RasterizerCachedMemory: {
585 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::FlushAndInvalidate); 649 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
650 FlushMode::FlushAndInvalidate);
586 std::memset(GetPointerFromVMA(current_vaddr), 0, copy_amount); 651 std::memset(GetPointerFromVMA(current_vaddr), 0, copy_amount);
587 break; 652 break;
588 } 653 }
589 case PageType::RasterizerCachedSpecial: { 654 case PageType::RasterizerCachedSpecial: {
590 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 655 DEBUG_ASSERT(GetMMIOHandler(current_vaddr));
591 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::FlushAndInvalidate); 656 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
657 FlushMode::FlushAndInvalidate);
592 GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, zeros.data(), copy_amount); 658 GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, zeros.data(), copy_amount);
593 break; 659 break;
594 } 660 }
@@ -609,11 +675,11 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) {
609 675
610 while (remaining_size > 0) { 676 while (remaining_size > 0) {
611 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size); 677 const size_t copy_amount = std::min(PAGE_SIZE - page_offset, remaining_size);
612 const VAddr current_vaddr = (page_index << PAGE_BITS) + page_offset; 678 const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
613 679
614 switch (current_page_table->attributes[page_index]) { 680 switch (current_page_table->attributes[page_index]) {
615 case PageType::Unmapped: { 681 case PageType::Unmapped: {
616 LOG_ERROR(HW_Memory, "unmapped CopyBlock @ 0x%llx (start address = 0x%llx, size = %zu)", 682 LOG_ERROR(HW_Memory, "unmapped CopyBlock @ 0x%08X (start address = 0x%08X, size = %zu)",
617 current_vaddr, src_addr, size); 683 current_vaddr, src_addr, size);
618 ZeroBlock(dest_addr, copy_amount); 684 ZeroBlock(dest_addr, copy_amount);
619 break; 685 break;
@@ -633,13 +699,15 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) {
633 break; 699 break;
634 } 700 }
635 case PageType::RasterizerCachedMemory: { 701 case PageType::RasterizerCachedMemory: {
636 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::Flush); 702 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
703 FlushMode::Flush);
637 WriteBlock(dest_addr, GetPointerFromVMA(current_vaddr), copy_amount); 704 WriteBlock(dest_addr, GetPointerFromVMA(current_vaddr), copy_amount);
638 break; 705 break;
639 } 706 }
640 case PageType::RasterizerCachedSpecial: { 707 case PageType::RasterizerCachedSpecial: {
641 DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); 708 DEBUG_ASSERT(GetMMIOHandler(current_vaddr));
642 RasterizerFlushVirtualRegion(current_vaddr, copy_amount, FlushMode::Flush); 709 RasterizerFlushVirtualRegion(current_vaddr, static_cast<u32>(copy_amount),
710 FlushMode::Flush);
643 711
644 std::vector<u8> buffer(copy_amount); 712 std::vector<u8> buffer(copy_amount);
645 GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, buffer.data(), buffer.size()); 713 GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, buffer.data(), buffer.size());
@@ -652,8 +720,8 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) {
652 720
653 page_index++; 721 page_index++;
654 page_offset = 0; 722 page_offset = 0;
655 dest_addr += copy_amount; 723 dest_addr += static_cast<VAddr>(copy_amount);
656 src_addr += copy_amount; 724 src_addr += static_cast<VAddr>(copy_amount);
657 remaining_size -= copy_amount; 725 remaining_size -= copy_amount;
658 } 726 }
659} 727}
@@ -721,7 +789,7 @@ boost::optional<PAddr> TryVirtualToPhysicalAddress(const VAddr addr) {
721PAddr VirtualToPhysicalAddress(const VAddr addr) { 789PAddr VirtualToPhysicalAddress(const VAddr addr) {
722 auto paddr = TryVirtualToPhysicalAddress(addr); 790 auto paddr = TryVirtualToPhysicalAddress(addr);
723 if (!paddr) { 791 if (!paddr) {
724 LOG_ERROR(HW_Memory, "Unknown virtual address @ 0x%llx", addr); 792 LOG_ERROR(HW_Memory, "Unknown virtual address @ 0x%08X", addr);
725 // To help with debugging, set bit on address so that it's obviously invalid. 793 // To help with debugging, set bit on address so that it's obviously invalid.
726 return addr | 0x80000000; 794 return addr | 0x80000000;
727 } 795 }
@@ -746,4 +814,4 @@ boost::optional<VAddr> PhysicalToVirtualAddress(const PAddr addr) {
746 return boost::none; 814 return boost::none;
747} 815}
748 816
749} // namespace 817} // namespace Memory