Merge remote-tracking branch 'upstream/master' into nx

# Conflicts:
#	src/core/CMakeLists.txt
#	src/core/arm/dynarmic/arm_dynarmic.cpp
#	src/core/arm/dyncom/arm_dyncom.cpp
#	src/core/hle/kernel/process.cpp
#	src/core/hle/kernel/thread.cpp
#	src/core/hle/kernel/thread.h
#	src/core/hle/kernel/vm_manager.cpp
#	src/core/loader/3dsx.cpp
#	src/core/loader/elf.cpp
#	src/core/loader/ncch.cpp
#	src/core/memory.cpp
#	src/core/memory.h
#	src/core/memory_setup.h

1 files changed, 105 insertions, 266 deletions

diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp
index 728886ea8..e33a37b2e 100644
--- a/src/core/loader/ncch.cpp
+++ b/src/core/loader/ncch.cpp
@@ -4,13 +4,17 @@
 
 #include <algorithm>
 #include <cinttypes>
+#include <codecvt>
 #include <cstring>
+#include <locale>
 #include <memory>
 #include "common/logging/log.h"
 #include "common/string_util.h"
 #include "common/swap.h"
 #include "core/core.h"
 #include "core/file_sys/archive_selfncch.h"
+#include "core/file_sys/ncch_container.h"
+#include "core/file_sys/title_metadata.h"
 #include "core/hle/kernel/process.h"
 #include "core/hle/kernel/resource_limit.h"
 #include "core/hle/service/cfg/cfg.h"
@@ -18,93 +22,14 @@
 #include "core/loader/ncch.h"
 #include "core/loader/smdh.h"
 #include "core/memory.h"
+#include "network/network.h"
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // Loader namespace
 
 namespace Loader {
 
-static const int kMaxSections = 8;   ///< Maximum number of sections (files) in an ExeFs
-static const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes)
-
-/**
- * Get the decompressed size of an LZSS compressed ExeFS file
- * @param buffer Buffer of compressed file
- * @param size Size of compressed buffer
- * @return Size of decompressed buffer
- */
-static u32 LZSS_GetDecompressedSize(const u8* buffer, u32 size) {
-    u32 offset_size = *(u32*)(buffer + size - 4);
-    return offset_size + size;
-}
-
-/**
- * Decompress ExeFS file (compressed with LZSS)
- * @param compressed Compressed buffer
- * @param compressed_size Size of compressed buffer
- * @param decompressed Decompressed buffer
- * @param decompressed_size Size of decompressed buffer
- * @return True on success, otherwise false
- */
-static bool LZSS_Decompress(const u8* compressed, u32 compressed_size, u8* decompressed,
-                            u32 decompressed_size) {
-    const u8* footer = compressed + compressed_size - 8;
-    u32 buffer_top_and_bottom = *reinterpret_cast<const u32*>(footer);
-    u32 out = decompressed_size;
-    u32 index = compressed_size - ((buffer_top_and_bottom >> 24) & 0xFF);
-    u32 stop_index = compressed_size - (buffer_top_and_bottom & 0xFFFFFF);
-
-    memset(decompressed, 0, decompressed_size);
-    memcpy(decompressed, compressed, compressed_size);
-
-    while (index > stop_index) {
-        u8 control = compressed[--index];
-
-        for (unsigned i = 0; i < 8; i++) {
-            if (index <= stop_index)
-                break;
-            if (index <= 0)
-                break;
-            if (out <= 0)
-                break;
-
-            if (control & 0x80) {
-                // Check if compression is out of bounds
-                if (index < 2)
-                    return false;
-                index -= 2;
-
-                u32 segment_offset = compressed[index] | (compressed[index + 1] << 8);
-                u32 segment_size = ((segment_offset >> 12) & 15) + 3;
-                segment_offset &= 0x0FFF;
-                segment_offset += 2;
-
-                // Check if compression is out of bounds
-                if (out < segment_size)
-                    return false;
-
-                for (unsigned j = 0; j < segment_size; j++) {
-                    // Check if compression is out of bounds
-                    if (out + segment_offset >= decompressed_size)
-                        return false;
-
-                    u8 data = decompressed[out + segment_offset];
-                    decompressed[--out] = data;
-                }
-            } else {
-                // Check if compression is out of bounds
-                if (out < 1)
-                    return false;
-                decompressed[--out] = compressed[--index];
-            }
-            control <<= 1;
-        }
-    }
-    return true;
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////////////
-// AppLoader_NCCH class
+static const u64 UPDATE_MASK = 0x0000000e00000000;
 
 FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) {
     u32 magic;
@@ -121,203 +46,105 @@ FileType AppLoader_NCCH::IdentifyType(FileUtil::IOFile& file) {
     return FileType::Error;
 }
 
+static std::string GetUpdateNCCHPath(u64_le program_id) {
+    u32 high = static_cast<u32>((program_id | UPDATE_MASK) >> 32);
+    u32 low = static_cast<u32>((program_id | UPDATE_MASK) & 0xFFFFFFFF);
+
+    // TODO(shinyquagsire23): Title database should be doing this path lookup
+    std::string content_path = Common::StringFromFormat(
+        "%sNintendo 3DS/%s/%s/title/%08x/%08x/content/", FileUtil::GetUserPath(D_SDMC_IDX).c_str(),
+        SYSTEM_ID, SDCARD_ID, high, low);
+    std::string tmd_path = content_path + "00000000.tmd";
+
+    u32 content_id = 0;
+    FileSys::TitleMetadata tmd(tmd_path);
+    if (tmd.Load() == ResultStatus::Success) {
+        content_id = tmd.GetBootContentID();
+    }
+
+    return Common::StringFromFormat("%s%08x.app", content_path.c_str(), content_id);
+}
+
 std::pair<boost::optional<u32>, ResultStatus> AppLoader_NCCH::LoadKernelSystemMode() {
     if (!is_loaded) {
-        ResultStatus res = LoadExeFS();
+        ResultStatus res = base_ncch.Load();
         if (res != ResultStatus::Success) {
             return std::make_pair(boost::none, res);
         }
     }
+
     // Set the system mode as the one from the exheader.
-    return std::make_pair(exheader_header.arm11_system_local_caps.system_mode.Value(),
+    return std::make_pair(overlay_ncch->exheader_header.arm11_system_local_caps.system_mode.Value(),
                           ResultStatus::Success);
 }
 
-ResultStatus AppLoader_NCCH::LoadExec() {
-    using Kernel::SharedPtr;
+ResultStatus AppLoader_NCCH::LoadExec(Kernel::SharedPtr<Kernel::Process>& process) {
     using Kernel::CodeSet;
+    using Kernel::SharedPtr;
 
     if (!is_loaded)
         return ResultStatus::ErrorNotLoaded;
 
     std::vector<u8> code;
-    if (ResultStatus::Success == ReadCode(code)) {
+    u64_le program_id;
+    if (ResultStatus::Success == ReadCode(code) &&
+        ResultStatus::Success == ReadProgramId(program_id)) {
         std::string process_name = Common::StringFromFixedZeroTerminatedBuffer(
-            (const char*)exheader_header.codeset_info.name, 8);
+            (const char*)overlay_ncch->exheader_header.codeset_info.name, 8);
 
-        SharedPtr<CodeSet> codeset = CodeSet::Create(process_name, ncch_header.program_id);
+        SharedPtr<CodeSet> codeset = CodeSet::Create(process_name, program_id);
 
         codeset->code.offset = 0;
-        codeset->code.addr = exheader_header.codeset_info.text.address;
-        codeset->code.size = exheader_header.codeset_info.text.num_max_pages * Memory::PAGE_SIZE;
+        codeset->code.addr = overlay_ncch->exheader_header.codeset_info.text.address;
+        codeset->code.size =
+            overlay_ncch->exheader_header.codeset_info.text.num_max_pages * Memory::PAGE_SIZE;
 
         codeset->rodata.offset = codeset->code.offset + codeset->code.size;
-        codeset->rodata.addr = exheader_header.codeset_info.ro.address;
-        codeset->rodata.size = exheader_header.codeset_info.ro.num_max_pages * Memory::PAGE_SIZE;
+        codeset->rodata.addr = overlay_ncch->exheader_header.codeset_info.ro.address;
+        codeset->rodata.size =
+            overlay_ncch->exheader_header.codeset_info.ro.num_max_pages * Memory::PAGE_SIZE;
 
         // TODO(yuriks): Not sure if the bss size is added to the page-aligned .data size or just
         //               to the regular size. Playing it safe for now.
-        u32 bss_page_size = (exheader_header.codeset_info.bss_size + 0xFFF) & ~0xFFF;
+        u32 bss_page_size = (overlay_ncch->exheader_header.codeset_info.bss_size + 0xFFF) & ~0xFFF;
         code.resize(code.size() + bss_page_size, 0);
 
         codeset->data.offset = codeset->rodata.offset + codeset->rodata.size;
-        codeset->data.addr = exheader_header.codeset_info.data.address;
+        codeset->data.addr = overlay_ncch->exheader_header.codeset_info.data.address;
         codeset->data.size =
-            exheader_header.codeset_info.data.num_max_pages * Memory::PAGE_SIZE + bss_page_size;
+            overlay_ncch->exheader_header.codeset_info.data.num_max_pages * Memory::PAGE_SIZE +
+            bss_page_size;
 
         codeset->entrypoint = codeset->code.addr;
         codeset->memory = std::make_shared<std::vector<u8>>(std::move(code));
 
-        Kernel::g_current_process = Kernel::Process::Create("main");
-        Kernel::g_current_process->LoadModule(codeset, codeset->entrypoint);
+        process = Kernel::Process::Create("main");
+        process->LoadModule(codeset, codeset->entrypoint);
 
         // Attach a resource limit to the process based on the resource limit category
-        Kernel::g_current_process->resource_limit =
+        process->resource_limit =
             Kernel::ResourceLimit::GetForCategory(static_cast<Kernel::ResourceLimitCategory>(
-                exheader_header.arm11_system_local_caps.resource_limit_category));
+                overlay_ncch->exheader_header.arm11_system_local_caps.resource_limit_category));
 
         // Set the default CPU core for this process
-        Kernel::g_current_process->ideal_processor =
-            exheader_header.arm11_system_local_caps.ideal_processor;
+        process->ideal_processor =
+            overlay_ncch->exheader_header.arm11_system_local_caps.ideal_processor;
 
         // Copy data while converting endianness
-        std::array<u32, ARRAY_SIZE(exheader_header.arm11_kernel_caps.descriptors)> kernel_caps;
-        std::copy_n(exheader_header.arm11_kernel_caps.descriptors, kernel_caps.size(),
+        std::array<u32, ARRAY_SIZE(overlay_ncch->exheader_header.arm11_kernel_caps.descriptors)>
+            kernel_caps;
+        std::copy_n(overlay_ncch->exheader_header.arm11_kernel_caps.descriptors, kernel_caps.size(),
                     begin(kernel_caps));
-        Kernel::g_current_process->ParseKernelCaps(kernel_caps.data(), kernel_caps.size());
+        process->ParseKernelCaps(kernel_caps.data(), kernel_caps.size());
 
-        s32 priority = exheader_header.arm11_system_local_caps.priority;
-        u32 stack_size = exheader_header.codeset_info.stack_size;
-        Kernel::g_current_process->Run(codeset->entrypoint, priority, stack_size);
+        s32 priority = overlay_ncch->exheader_header.arm11_system_local_caps.priority;
+        u32 stack_size = overlay_ncch->exheader_header.codeset_info.stack_size;
+        process->Run(codeset->entrypoint, priority, stack_size);
         return ResultStatus::Success;
     }
     return ResultStatus::Error;
 }
 
-ResultStatus AppLoader_NCCH::LoadSectionExeFS(const char* name, std::vector<u8>& buffer) {
-    if (!file.IsOpen())
-        return ResultStatus::Error;
-
-    ResultStatus result = LoadExeFS();
-    if (result != ResultStatus::Success)
-        return result;
-
-    LOG_DEBUG(Loader, "%d sections:", kMaxSections);
-    // Iterate through the ExeFs archive until we find a section with the specified name...
-    for (unsigned section_number = 0; section_number < kMaxSections; section_number++) {
-        const auto& section = exefs_header.section[section_number];
-
-        // Load the specified section...
-        if (strcmp(section.name, name) == 0) {
-            LOG_DEBUG(Loader, "%d - offset: 0x%08X, size: 0x%08X, name: %s", section_number,
-                      section.offset, section.size, section.name);
-
-            s64 section_offset =
-                (section.offset + exefs_offset + sizeof(ExeFs_Header) + ncch_offset);
-            file.Seek(section_offset, SEEK_SET);
-
-            if (strcmp(section.name, ".code") == 0 && is_compressed) {
-                // Section is compressed, read compressed .code section...
-                std::unique_ptr<u8[]> temp_buffer;
-                try {
-                    temp_buffer.reset(new u8[section.size]);
-                } catch (std::bad_alloc&) {
-                    return ResultStatus::ErrorMemoryAllocationFailed;
-                }
-
-                if (file.ReadBytes(&temp_buffer[0], section.size) != section.size)
-                    return ResultStatus::Error;
-
-                // Decompress .code section...
-                u32 decompressed_size = LZSS_GetDecompressedSize(&temp_buffer[0], section.size);
-                buffer.resize(decompressed_size);
-                if (!LZSS_Decompress(&temp_buffer[0], section.size, &buffer[0], decompressed_size))
-                    return ResultStatus::ErrorInvalidFormat;
-            } else {
-                // Section is uncompressed...
-                buffer.resize(section.size);
-                if (file.ReadBytes(&buffer[0], section.size) != section.size)
-                    return ResultStatus::Error;
-            }
-            return ResultStatus::Success;
-        }
-    }
-    return ResultStatus::ErrorNotUsed;
-}
-
-ResultStatus AppLoader_NCCH::LoadExeFS() {
-    if (is_exefs_loaded)
-        return ResultStatus::Success;
-
-    if (!file.IsOpen())
-        return ResultStatus::Error;
-
-    // Reset read pointer in case this file has been read before.
-    file.Seek(0, SEEK_SET);
-
-    if (file.ReadBytes(&ncch_header, sizeof(NCCH_Header)) != sizeof(NCCH_Header))
-        return ResultStatus::Error;
-
-    // Skip NCSD header and load first NCCH (NCSD is just a container of NCCH files)...
-    if (MakeMagic('N', 'C', 'S', 'D') == ncch_header.magic) {
-        LOG_DEBUG(Loader, "Only loading the first (bootable) NCCH within the NCSD file!");
-        ncch_offset = 0x4000;
-        file.Seek(ncch_offset, SEEK_SET);
-        file.ReadBytes(&ncch_header, sizeof(NCCH_Header));
-    }
-
-    // Verify we are loading the correct file type...
-    if (MakeMagic('N', 'C', 'C', 'H') != ncch_header.magic)
-        return ResultStatus::ErrorInvalidFormat;
-
-    // Read ExHeader...
-
-    if (file.ReadBytes(&exheader_header, sizeof(ExHeader_Header)) != sizeof(ExHeader_Header))
-        return ResultStatus::Error;
-
-    is_compressed = (exheader_header.codeset_info.flags.flag & 1) == 1;
-    entry_point = exheader_header.codeset_info.text.address;
-    code_size = exheader_header.codeset_info.text.code_size;
-    stack_size = exheader_header.codeset_info.stack_size;
-    bss_size = exheader_header.codeset_info.bss_size;
-    core_version = exheader_header.arm11_system_local_caps.core_version;
-    priority = exheader_header.arm11_system_local_caps.priority;
-    resource_limit_category = exheader_header.arm11_system_local_caps.resource_limit_category;
-
-    LOG_DEBUG(Loader, "Name:                        %s", exheader_header.codeset_info.name);
-    LOG_DEBUG(Loader, "Program ID:                  %016" PRIX64, ncch_header.program_id);
-    LOG_DEBUG(Loader, "Code compressed:             %s", is_compressed ? "yes" : "no");
-    LOG_DEBUG(Loader, "Entry point:                 0x%08X", entry_point);
-    LOG_DEBUG(Loader, "Code size:                   0x%08X", code_size);
-    LOG_DEBUG(Loader, "Stack size:                  0x%08X", stack_size);
-    LOG_DEBUG(Loader, "Bss size:                    0x%08X", bss_size);
-    LOG_DEBUG(Loader, "Core version:                %d", core_version);
-    LOG_DEBUG(Loader, "Thread priority:             0x%X", priority);
-    LOG_DEBUG(Loader, "Resource limit category:     %d", resource_limit_category);
-    LOG_DEBUG(Loader, "System Mode:                 %d",
-              static_cast<int>(exheader_header.arm11_system_local_caps.system_mode));
-
-    if (exheader_header.arm11_system_local_caps.program_id != ncch_header.program_id) {
-        LOG_ERROR(Loader, "ExHeader Program ID mismatch: the ROM is probably encrypted.");
-        return ResultStatus::ErrorEncrypted;
-    }
-
-    // Read ExeFS...
-
-    exefs_offset = ncch_header.exefs_offset * kBlockSize;
-    u32 exefs_size = ncch_header.exefs_size * kBlockSize;
-
-    LOG_DEBUG(Loader, "ExeFS offset:                0x%08X", exefs_offset);
-    LOG_DEBUG(Loader, "ExeFS size:                  0x%08X", exefs_size);
-
-    file.Seek(exefs_offset + ncch_offset, SEEK_SET);
-    if (file.ReadBytes(&exefs_header, sizeof(ExeFs_Header)) != sizeof(ExeFs_Header))
-        return ResultStatus::Error;
-
-    is_exefs_loaded = true;
-    return ResultStatus::Success;
-}
-
 void AppLoader_NCCH::ParseRegionLockoutInfo() {
     std::vector<u8> smdh_buffer;
     if (ReadIcon(smdh_buffer) == ResultStatus::Success && smdh_buffer.size() >= sizeof(SMDH)) {
@@ -335,28 +162,43 @@ void AppLoader_NCCH::ParseRegionLockoutInfo() {
     }
 }
 
-ResultStatus AppLoader_NCCH::Load() {
+ResultStatus AppLoader_NCCH::Load(Kernel::SharedPtr<Kernel::Process>& process) {
+    u64_le ncch_program_id;
+
     if (is_loaded)
         return ResultStatus::ErrorAlreadyLoaded;
 
-    ResultStatus result = LoadExeFS();
+    ResultStatus result = base_ncch.Load();
     if (result != ResultStatus::Success)
         return result;
 
-    std::string program_id{Common::StringFromFormat("%016" PRIX64, ncch_header.program_id)};
+    ReadProgramId(ncch_program_id);
+    std::string program_id{Common::StringFromFormat("%016" PRIX64, ncch_program_id)};
 
     LOG_INFO(Loader, "Program ID: %s", program_id.c_str());
 
+    update_ncch.OpenFile(GetUpdateNCCHPath(ncch_program_id));
+    result = update_ncch.Load();
+    if (result == ResultStatus::Success) {
+        overlay_ncch = &update_ncch;
+    }
+
     Core::Telemetry().AddField(Telemetry::FieldType::Session, "ProgramId", program_id);
 
+    if (auto room_member = Network::GetRoomMember().lock()) {
+        Network::GameInfo game_info;
+        ReadTitle(game_info.name);
+        game_info.id = ncch_program_id;
+        room_member->SendGameInfo(game_info);
+    }
+
     is_loaded = true; // Set state to loaded
 
-    result = LoadExec(); // Load the executable into memory for booting
+    result = LoadExec(process); // Load the executable into memory for booting
     if (ResultStatus::Success != result)
         return result;
 
-    Service::FS::RegisterArchiveType(std::make_unique<FileSys::ArchiveFactory_SelfNCCH>(*this),
-                                     Service::FS::ArchiveIdCode::SelfNCCH);
+    Service::FS::RegisterSelfNCCH(*this);
 
     ParseRegionLockoutInfo();
 
@@ -364,61 +206,58 @@ ResultStatus AppLoader_NCCH::Load() {
 }
 
 ResultStatus AppLoader_NCCH::ReadCode(std::vector<u8>& buffer) {
-    return LoadSectionExeFS(".code", buffer);
+    return overlay_ncch->LoadSectionExeFS(".code", buffer);
 }
 
 ResultStatus AppLoader_NCCH::ReadIcon(std::vector<u8>& buffer) {
-    return LoadSectionExeFS("icon", buffer);
+    return overlay_ncch->LoadSectionExeFS("icon", buffer);
 }
 
 ResultStatus AppLoader_NCCH::ReadBanner(std::vector<u8>& buffer) {
-    return LoadSectionExeFS("banner", buffer);
+    return overlay_ncch->LoadSectionExeFS("banner", buffer);
 }
 
 ResultStatus AppLoader_NCCH::ReadLogo(std::vector<u8>& buffer) {
-    return LoadSectionExeFS("logo", buffer);
+    return overlay_ncch->LoadSectionExeFS("logo", buffer);
 }
 
 ResultStatus AppLoader_NCCH::ReadProgramId(u64& out_program_id) {
-    if (!file.IsOpen())
-        return ResultStatus::Error;
-
-    ResultStatus result = LoadExeFS();
+    ResultStatus result = base_ncch.ReadProgramId(out_program_id);
     if (result != ResultStatus::Success)
         return result;
 
-    out_program_id = ncch_header.program_id;
     return ResultStatus::Success;
 }
 
 ResultStatus AppLoader_NCCH::ReadRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file, u64& offset,
                                        u64& size) {
-    if (!file.IsOpen())
-        return ResultStatus::Error;
+    return base_ncch.ReadRomFS(romfs_file, offset, size);
+}
 
-    // Check if the NCCH has a RomFS...
-    if (ncch_header.romfs_offset != 0 && ncch_header.romfs_size != 0) {
-        u32 romfs_offset = ncch_offset + (ncch_header.romfs_offset * kBlockSize) + 0x1000;
-        u32 romfs_size = (ncch_header.romfs_size * kBlockSize) - 0x1000;
+ResultStatus AppLoader_NCCH::ReadUpdateRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file,
+                                             u64& offset, u64& size) {
+    ResultStatus result = update_ncch.ReadRomFS(romfs_file, offset, size);
 
-        LOG_DEBUG(Loader, "RomFS offset:           0x%08X", romfs_offset);
-        LOG_DEBUG(Loader, "RomFS size:             0x%08X", romfs_size);
+    if (result != ResultStatus::Success)
+        return base_ncch.ReadRomFS(romfs_file, offset, size);
+}
 
-        if (file.GetSize() < romfs_offset + romfs_size)
-            return ResultStatus::Error;
+ResultStatus AppLoader_NCCH::ReadTitle(std::string& title) {
+    std::vector<u8> data;
+    Loader::SMDH smdh;
+    ReadIcon(data);
 
-        // We reopen the file, to allow its position to be independent from file's
-        romfs_file = std::make_shared<FileUtil::IOFile>(filepath, "rb");
-        if (!romfs_file->IsOpen())
-            return ResultStatus::Error;
+    if (!Loader::IsValidSMDH(data)) {
+        return ResultStatus::ErrorInvalidFormat;
+    }
 
-        offset = romfs_offset;
-        size = romfs_size;
+    memcpy(&smdh, data.data(), sizeof(Loader::SMDH));
 
-        return ResultStatus::Success;
-    }
-    LOG_DEBUG(Loader, "NCCH has no RomFS");
-    return ResultStatus::ErrorNotUsed;
+    const auto& short_title = smdh.GetShortTitle(SMDH::TitleLanguage::English);
+    auto title_end = std::find(short_title.begin(), short_title.end(), u'\0');
+    title = Common::UTF16ToUTF8(std::u16string{short_title.begin(), title_end});
+
+    return ResultStatus::Success;
 }
 
 } // namespace Loader