startup_checks: Use Windows flow for *nix

Spawns a child using fork and exec as opposed to fork alone. Workaround
for the macos file manager complaining about not supporting fork without
exec.

Control flow for *nix is now roughly the same as for Windows.

2 files changed, 51 insertions, 9 deletions

diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
index 6a91212e2..95c9fdecb 100644
--- a/src/yuzu/startup_checks.cpp
+++ b/src/yuzu/startup_checks.cpp
@@ -4,11 +4,14 @@
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
 #ifdef _WIN32
-#include <cstring> // for memset, strncpy
+#include <cstring> // for memset, strncpy, strncmp
 #include <processthreadsapi.h>
 #include <windows.h>
 #elif defined(YUZU_UNIX)
+#include <cstring> // for strncmp
 #include <errno.h>
+#include <spawn.h>
+#include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
 #endif
@@ -53,6 +56,13 @@ bool CheckEnvVars(bool* is_child) {
                      IS_CHILD_ENV_VAR, GetLastError());
         return true;
     }
+#elif defined(YUZU_UNIX)
+    const char* startup_check_var = getenv(STARTUP_CHECK_ENV_VAR);
+    if (startup_check_var != nullptr &&
+        std::strncmp(startup_check_var, ENV_VAR_ENABLED_TEXT, 8) == 0) {
+        CheckVulkan();
+        return true;
+    }
 #endif
     return false;
 }
@@ -101,20 +111,22 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
     }
 
 #elif defined(YUZU_UNIX)
+    const int env_var_set = setenv(STARTUP_CHECK_ENV_VAR, ENV_VAR_ENABLED_TEXT, 1);
+    if (env_var_set == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "setenv failed to set %s with error %d\n", STARTUP_CHECK_ENV_VAR, err);
+        return false;
+    }
+
     if (perform_vulkan_check) {
-        const pid_t pid = fork();
-        if (pid == 0) {
-            CheckVulkan();
-            return true;
-        } else if (pid == -1) {
-            const int err = errno;
-            std::fprintf(stderr, "fork failed with error %d\n", err);
+        const pid_t pid = SpawnChild(arg0);
+        if (pid == -1) {
             return false;
         }
 
         // Get exit code from child process
         int status;
-        const int r_val = wait(&status);
+        const int r_val = waitpid(pid, &status, 0);
         if (r_val == -1) {
             const int err = errno;
             std::fprintf(stderr, "wait failed with error %d\n", err);
@@ -123,6 +135,13 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
         // Vulkan is broken if the child crashed (return value is not zero)
         *has_broken_vulkan = (status != 0);
     }
+
+    const int env_var_cleared = unsetenv(STARTUP_CHECK_ENV_VAR);
+    if (env_var_cleared == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "unsetenv failed to clear %s with error %d\n", STARTUP_CHECK_ENV_VAR,
+                     err);
+    }
 #endif
     return false;
 }
@@ -156,4 +175,23 @@ bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags) {
 
     return true;
 }
+#elif defined(YUZU_UNIX)
+pid_t SpawnChild(const char* arg0) {
+    const pid_t pid = fork();
+
+    if (pid == -1) {
+        // error
+        const int err = errno;
+        std::fprintf(stderr, "fork failed with error %d\n", err);
+        return pid;
+    } else if (pid == 0) {
+        // child
+        execl(arg0, arg0, nullptr);
+        const int err = errno;
+        std::fprintf(stderr, "execl failed with error %d\n", err);
+        return -1;
+    }
+
+    return pid;
+}
 #endif
diff --git a/src/yuzu/startup_checks.h b/src/yuzu/startup_checks.h
index d8e563be6..2f86fb843 100644
--- a/src/yuzu/startup_checks.h
+++ b/src/yuzu/startup_checks.h
@@ -5,6 +5,8 @@
 
 #ifdef _WIN32
 #include <windows.h>
+#elif defined(YUZU_UNIX)
+#include <sys/types.h>
 #endif
 
 constexpr char IS_CHILD_ENV_VAR[] = "YUZU_IS_CHILD";
@@ -17,4 +19,6 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
 
 #ifdef _WIN32
 bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags);
+#elif defined(YUZU_UNIX)
+pid_t SpawnChild(const char* arg0);
 #endif