nso: Return an optional address from LoadModule

If a malformed NSO is attempted to be loaded, we shouldn't continue
onwards. We should be reporting an error and bailing out.

5 files changed, 29 insertions, 16 deletions

diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp
index 5a8bb4675..8518dddcb 100644
--- a/src/core/loader/deconstructed_rom_directory.cpp
+++ b/src/core/loader/deconstructed_rom_directory.cpp
@@ -139,14 +139,22 @@ ResultStatus AppLoader_DeconstructedRomDirectory::Load(Kernel::Process& process)
     for (const auto& module : {"rtld", "main", "subsdk0", "subsdk1", "subsdk2", "subsdk3",
                                "subsdk4", "subsdk5", "subsdk6", "subsdk7", "sdk"}) {
         const FileSys::VirtualFile module_file = dir->GetFile(module);
-        if (module_file != nullptr) {
-            const VAddr load_addr = next_load_addr;
-            next_load_addr = AppLoader_NSO::LoadModule(*module_file, load_addr,
-                                                       std::strcmp(module, "rtld") == 0, pm);
-            LOG_DEBUG(Loader, "loaded module {} @ 0x{:X}", module, load_addr);
-            // Register module with GDBStub
-            GDBStub::RegisterModule(module, load_addr, next_load_addr - 1, false);
+        if (module_file == nullptr) {
+            continue;
         }
+
+        const VAddr load_addr = next_load_addr;
+        const bool should_pass_arguments = std::strcmp(module, "rtld") == 0;
+        const auto tentative_next_load_addr =
+            AppLoader_NSO::LoadModule(*module_file, load_addr, should_pass_arguments, pm);
+        if (!tentative_next_load_addr) {
+            return ResultStatus::ErrorLoadingNSO;
+        }
+
+        next_load_addr = *tentative_next_load_addr;
+        LOG_DEBUG(Loader, "loaded module {} @ 0x{:X}", module, load_addr);
+        // Register module with GDBStub
+        GDBStub::RegisterModule(module, load_addr, next_load_addr - 1, false);
     }
 
     process.Run(base_address, metadata.GetMainThreadPriority(), metadata.GetMainThreadStackSize());
diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp
index 91659ec17..9cd0b0ccd 100644
--- a/src/core/loader/loader.cpp
+++ b/src/core/loader/loader.cpp
@@ -93,7 +93,7 @@ std::string GetFileTypeString(FileType type) {
     return "unknown";
 }
 
-constexpr std::array<const char*, 59> RESULT_MESSAGES{
+constexpr std::array<const char*, 60> RESULT_MESSAGES{
     "The operation completed successfully.",
     "The loader requested to load is already loaded.",
     "The operation is not implemented.",
@@ -128,6 +128,7 @@ constexpr std::array<const char*, 59> RESULT_MESSAGES{
     "The RomFS could not be found.",
     "The ELF file has incorrect size as determined by the header.",
     "There was a general error loading the NRO into emulated memory.",
+    "There was a general error loading the NSO into emulated memory.",
     "There is no icon available.",
     "There is no control data available.",
     "The NAX file has a bad header.",
diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h
index 0e0333db5..e562b3a04 100644
--- a/src/core/loader/loader.h
+++ b/src/core/loader/loader.h
@@ -90,6 +90,7 @@ enum class ResultStatus : u16 {
     ErrorNoRomFS,
     ErrorIncorrectELFFileSize,
     ErrorLoadingNRO,
+    ErrorLoadingNSO,
     ErrorNoIcon,
     ErrorNoControl,
     ErrorBadNAXHeader,
diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp
index d26fa482c..68efca5c0 100644
--- a/src/core/loader/nso.cpp
+++ b/src/core/loader/nso.cpp
@@ -93,9 +93,9 @@ static constexpr u32 PageAlignSize(u32 size) {
     return (size + Memory::PAGE_MASK) & ~Memory::PAGE_MASK;
 }
 
-VAddr AppLoader_NSO::LoadModule(const FileSys::VfsFile& file, VAddr load_base,
-                                bool should_pass_arguments,
-                                boost::optional<FileSys::PatchManager> pm) {
+std::optional<VAddr> AppLoader_NSO::LoadModule(const FileSys::VfsFile& file, VAddr load_base,
+                                               bool should_pass_arguments,
+                                               std::optional<FileSys::PatchManager> pm) {
     if (file.GetSize() < sizeof(NsoHeader))
         return {};
 
@@ -154,7 +154,7 @@ VAddr AppLoader_NSO::LoadModule(const FileSys::VfsFile& file, VAddr load_base,
     program_image.resize(image_size);
 
     // Apply patches if necessary
-    if (pm != boost::none && pm->HasNSOPatch(nso_header.build_id)) {
+    if (pm && pm->HasNSOPatch(nso_header.build_id)) {
         std::vector<u8> pi_header(program_image.size() + 0x100);
         std::memcpy(pi_header.data(), &nso_header, sizeof(NsoHeader));
         std::memcpy(pi_header.data() + 0x100, program_image.data(), program_image.size());
@@ -181,7 +181,9 @@ ResultStatus AppLoader_NSO::Load(Kernel::Process& process) {
 
     // Load module
     const VAddr base_address = process.VMManager().GetCodeRegionBaseAddress();
-    LoadModule(*file, base_address, true);
+    if (!LoadModule(*file, base_address, true)) {
+        return ResultStatus::ErrorLoadingNSO;
+    }
     LOG_DEBUG(Loader, "loaded module {} @ 0x{:X}", file->GetName(), base_address);
 
     process.Run(base_address, Kernel::THREADPRIO_DEFAULT, Memory::DEFAULT_STACK_SIZE);
diff --git a/src/core/loader/nso.h b/src/core/loader/nso.h
index d92897130..433306139 100644
--- a/src/core/loader/nso.h
+++ b/src/core/loader/nso.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <optional>
 #include "common/common_types.h"
 #include "core/file_sys/patch_manager.h"
 #include "core/loader/linker.h"
@@ -36,9 +37,9 @@ public:
         return IdentifyType(file);
     }
 
-    static VAddr LoadModule(const FileSys::VfsFile& file, VAddr load_base,
-                            bool should_pass_arguments,
-                            boost::optional<FileSys::PatchManager> pm = boost::none);
+    static std::optional<VAddr> LoadModule(const FileSys::VfsFile& file, VAddr load_base,
+                                           bool should_pass_arguments,
+                                           std::optional<FileSys::PatchManager> pm = {});
 
     ResultStatus Load(Kernel::Process& process) override;
 };