Merge pull request #8564 from lat9nq/dinner-fork

yuzu: Streamline broken Vulkan handling

12 files changed, 181 insertions, 124 deletions

diff --git a/src/yuzu/CMakeLists.txt b/src/yuzu/CMakeLists.txt
index a11a3b908..57e0e7025 100644
--- a/src/yuzu/CMakeLists.txt
+++ b/src/yuzu/CMakeLists.txt
@@ -30,8 +30,6 @@ add_executable(yuzu
     applets/qt_web_browser_scripts.h
     bootmanager.cpp
     bootmanager.h
-    check_vulkan.cpp
-    check_vulkan.h
     compatdb.ui
     compatibility_list.cpp
     compatibility_list.h
@@ -158,6 +156,8 @@ add_executable(yuzu
     main.cpp
     main.h
     main.ui
+    startup_checks.cpp
+    startup_checks.h
     uisettings.cpp
     uisettings.h
     util/controller_navigation.cpp
diff --git a/src/yuzu/check_vulkan.cpp b/src/yuzu/check_vulkan.cpp
deleted file mode 100644
index e6d66ab34..000000000
--- a/src/yuzu/check_vulkan.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include "video_core/vulkan_common/vulkan_wrapper.h"
-
-#include <filesystem>
-#include <fstream>
-#include "common/fs/fs.h"
-#include "common/fs/path_util.h"
-#include "common/logging/log.h"
-#include "video_core/vulkan_common/vulkan_instance.h"
-#include "video_core/vulkan_common/vulkan_library.h"
-#include "yuzu/check_vulkan.h"
-#include "yuzu/uisettings.h"
-
-constexpr char TEMP_FILE_NAME[] = "vulkan_check";
-
-bool CheckVulkan() {
-    if (UISettings::values.has_broken_vulkan) {
-        return true;
-    }
-
-    LOG_DEBUG(Frontend, "Checking presence of Vulkan");
-
-    const auto fs_config_loc = Common::FS::GetYuzuPath(Common::FS::YuzuPath::ConfigDir);
-    const auto temp_file_loc = fs_config_loc / TEMP_FILE_NAME;
-
-    if (std::filesystem::exists(temp_file_loc)) {
-        LOG_WARNING(Frontend, "Detected recovery from previous failed Vulkan initialization");
-
-        UISettings::values.has_broken_vulkan = true;
-        std::filesystem::remove(temp_file_loc);
-        return false;
-    }
-
-    std::ofstream temp_file_handle(temp_file_loc);
-    temp_file_handle.close();
-
-    try {
-        Vulkan::vk::InstanceDispatch dld;
-        const Common::DynamicLibrary library = Vulkan::OpenLibrary();
-        const Vulkan::vk::Instance instance =
-            Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_0);
-
-    } catch (const Vulkan::vk::Exception& exception) {
-        LOG_ERROR(Frontend, "Failed to initialize Vulkan: {}", exception.what());
-        // Don't set has_broken_vulkan to true here: we care when loading Vulkan crashes the
-        // application, not when we can handle it.
-    }
-
-    std::filesystem::remove(temp_file_loc);
-    return true;
-}
diff --git a/src/yuzu/check_vulkan.h b/src/yuzu/check_vulkan.h
deleted file mode 100644
index e4ea93582..000000000
--- a/src/yuzu/check_vulkan.h
+++ /dev/null
@@ -1,6 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#pragma once
-
-bool CheckVulkan();
diff --git a/src/yuzu/configuration/config.cpp b/src/yuzu/configuration/config.cpp
index 1f76e86b9..c841843f0 100644
--- a/src/yuzu/configuration/config.cpp
+++ b/src/yuzu/configuration/config.cpp
@@ -688,12 +688,6 @@ void Config::ReadRendererValues() {
     ReadGlobalSetting(Settings::values.bg_green);
     ReadGlobalSetting(Settings::values.bg_blue);
 
-    if (!global && UISettings::values.has_broken_vulkan &&
-        Settings::values.renderer_backend.GetValue() == Settings::RendererBackend::Vulkan &&
-        !Settings::values.renderer_backend.UsingGlobal()) {
-        Settings::values.renderer_backend.SetGlobal(true);
-    }
-
     if (global) {
         ReadBasicSetting(Settings::values.renderer_debug);
         ReadBasicSetting(Settings::values.renderer_shader_feedback);
@@ -813,7 +807,6 @@ void Config::ReadUIValues() {
     ReadBasicSetting(UISettings::values.pause_when_in_background);
     ReadBasicSetting(UISettings::values.mute_when_in_background);
     ReadBasicSetting(UISettings::values.hide_mouse);
-    ReadBasicSetting(UISettings::values.has_broken_vulkan);
     ReadBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
@@ -1367,7 +1360,6 @@ void Config::SaveUIValues() {
     WriteBasicSetting(UISettings::values.pause_when_in_background);
     WriteBasicSetting(UISettings::values.mute_when_in_background);
     WriteBasicSetting(UISettings::values.hide_mouse);
-    WriteBasicSetting(UISettings::values.has_broken_vulkan);
     WriteBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp
index 85f34dc35..6b33c4535 100644
--- a/src/yuzu/configuration/configure_graphics.cpp
+++ b/src/yuzu/configuration/configure_graphics.cpp
@@ -58,24 +58,9 @@ ConfigureGraphics::ConfigureGraphics(const Core::System& system_, QWidget* paren
         UpdateBackgroundColorButton(new_bg_color);
     });
 
-    connect(ui->button_check_vulkan, &QAbstractButton::clicked, this, [this] {
-        UISettings::values.has_broken_vulkan = false;
-
-        if (RetrieveVulkanDevices()) {
-            ui->api->setEnabled(true);
-            ui->button_check_vulkan->hide();
-
-            for (const auto& device : vulkan_devices) {
-                ui->device->addItem(device);
-            }
-        } else {
-            UISettings::values.has_broken_vulkan = true;
-        }
-    });
-
-    ui->api->setEnabled(!UISettings::values.has_broken_vulkan.GetValue());
-    ui->button_check_vulkan->setVisible(UISettings::values.has_broken_vulkan.GetValue());
-
+    ui->api->setEnabled(!UISettings::values.has_broken_vulkan);
+    ui->api_widget->setEnabled(!UISettings::values.has_broken_vulkan ||
+                               Settings::IsConfiguringGlobal());
     ui->bg_label->setVisible(Settings::IsConfiguringGlobal());
     ui->bg_combobox->setVisible(!Settings::IsConfiguringGlobal());
 }
@@ -315,7 +300,7 @@ void ConfigureGraphics::UpdateAPILayout() {
         vulkan_device = Settings::values.vulkan_device.GetValue(true);
         shader_backend = Settings::values.shader_backend.GetValue(true);
         ui->device_widget->setEnabled(false);
-        ui->backend_widget->setEnabled(UISettings::values.has_broken_vulkan.GetValue());
+        ui->backend_widget->setEnabled(false);
     } else {
         vulkan_device = Settings::values.vulkan_device.GetValue();
         shader_backend = Settings::values.shader_backend.GetValue();
@@ -337,9 +322,9 @@ void ConfigureGraphics::UpdateAPILayout() {
     }
 }
 
-bool ConfigureGraphics::RetrieveVulkanDevices() try {
+void ConfigureGraphics::RetrieveVulkanDevices() try {
     if (UISettings::values.has_broken_vulkan) {
-        return false;
+        return;
     }
 
     using namespace Vulkan;
@@ -355,11 +340,8 @@ bool ConfigureGraphics::RetrieveVulkanDevices() try {
         const std::string name = vk::PhysicalDevice(device, dld).GetProperties().deviceName;
         vulkan_devices.push_back(QString::fromStdString(name));
     }
-
-    return true;
 } catch (const Vulkan::vk::Exception& exception) {
     LOG_ERROR(Frontend, "Failed to enumerate devices with error: {}", exception.what());
-    return false;
 }
 
 Settings::RendererBackend ConfigureGraphics::GetCurrentGraphicsBackend() const {
@@ -440,11 +422,4 @@ void ConfigureGraphics::SetupPerGameUI() {
         ui->api, static_cast<int>(Settings::values.renderer_backend.GetValue(true)));
     ConfigurationShared::InsertGlobalItem(
         ui->nvdec_emulation, static_cast<int>(Settings::values.nvdec_emulation.GetValue(true)));
-
-    if (UISettings::values.has_broken_vulkan) {
-        ui->backend_widget->setEnabled(true);
-        ConfigurationShared::SetColoredComboBox(
-            ui->backend, ui->backend_widget,
-            static_cast<int>(Settings::values.shader_backend.GetValue(true)));
-    }
 }
diff --git a/src/yuzu/configuration/configure_graphics.h b/src/yuzu/configuration/configure_graphics.h
index 8438f0187..1b101c940 100644
--- a/src/yuzu/configuration/configure_graphics.h
+++ b/src/yuzu/configuration/configure_graphics.h
@@ -41,7 +41,7 @@ private:
     void UpdateDeviceSelection(int device);
     void UpdateShaderBackendSelection(int backend);
 
-    bool RetrieveVulkanDevices();
+    void RetrieveVulkanDevices();
 
     void SetupPerGameUI();
 
diff --git a/src/yuzu/configuration/configure_graphics.ui b/src/yuzu/configuration/configure_graphics.ui
index 2f94c94bc..1e4f74704 100644
--- a/src/yuzu/configuration/configure_graphics.ui
+++ b/src/yuzu/configuration/configure_graphics.ui
@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>471</width>
+    <width>541</width>
     <height>759</height>
    </rect>
   </property>
@@ -574,13 +574,6 @@
      </property>
     </spacer>
    </item>
-   <item>
-    <widget class="QPushButton" name="button_check_vulkan">
-     <property name="text">
-      <string>Check for Working Vulkan</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources/>
diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp
index 08ccc1555..2814548eb 100644
--- a/src/yuzu/main.cpp
+++ b/src/yuzu/main.cpp
@@ -115,7 +115,6 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual
 #include "video_core/shader_notify.h"
 #include "yuzu/about_dialog.h"
 #include "yuzu/bootmanager.h"
-#include "yuzu/check_vulkan.h"
 #include "yuzu/compatdb.h"
 #include "yuzu/compatibility_list.h"
 #include "yuzu/configuration/config.h"
@@ -131,6 +130,7 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual
 #include "yuzu/install_dialog.h"
 #include "yuzu/loading_screen.h"
 #include "yuzu/main.h"
+#include "yuzu/startup_checks.h"
 #include "yuzu/uisettings.h"
 
 using namespace Common::Literals;
@@ -252,7 +252,7 @@ static QString PrettyProductName() {
     return QSysInfo::prettyProductName();
 }
 
-GMainWindow::GMainWindow()
+GMainWindow::GMainWindow(bool has_broken_vulkan)
     : ui{std::make_unique<Ui::MainWindow>()}, system{std::make_unique<Core::System>()},
       input_subsystem{std::make_shared<InputCommon::InputSubsystem>()},
       config{std::make_unique<Config>(*system)},
@@ -352,17 +352,15 @@ GMainWindow::GMainWindow()
 
     MigrateConfigFiles();
 
-    if (!CheckVulkan()) {
-        config->Save();
+    if (has_broken_vulkan) {
+        UISettings::values.has_broken_vulkan = true;
+
+        QMessageBox::warning(this, tr("Broken Vulkan Installation Detected"),
+                             tr("Vulkan initialization failed during boot.<br><br>Click <a "
+                                "href='https://yuzu-emu.org/wiki/faq/"
+                                "#yuzu-starts-with-the-error-broken-vulkan-installation-detected'>"
+                                "here for instructions to fix the issue</a>."));
 
-        QMessageBox::warning(
-            this, tr("Broken Vulkan Installation Detected"),
-            tr("Vulkan initialization failed on the previous boot.<br><br>Click <a "
-               "href='https://yuzu-emu.org/wiki/faq/"
-               "#yuzu-starts-with-the-error-broken-vulkan-installation-detected'>here for "
-               "instructions to fix the issue</a>."));
-    }
-    if (UISettings::values.has_broken_vulkan) {
         Settings::values.renderer_backend = Settings::RendererBackend::OpenGL;
 
         renderer_status_button->setDisabled(true);
@@ -3879,6 +3877,11 @@ void GMainWindow::SetDiscordEnabled([[maybe_unused]] bool state) {
 #endif
 
 int main(int argc, char* argv[]) {
+    bool has_broken_vulkan = false;
+    if (StartupChecks(argv[0], &has_broken_vulkan)) {
+        return 0;
+    }
+
     Common::DetachedTasks detached_tasks;
     MicroProfileOnThreadCreate("Frontend");
     SCOPE_EXIT({ MicroProfileShutdown(); });
@@ -3918,7 +3921,7 @@ int main(int argc, char* argv[]) {
     // generating shaders
     setlocale(LC_ALL, "C");
 
-    GMainWindow main_window{};
+    GMainWindow main_window{has_broken_vulkan};
     // After settings have been loaded by GMainWindow, apply the filter
     main_window.show();
 
diff --git a/src/yuzu/main.h b/src/yuzu/main.h
index 09e37f152..27204f5a2 100644
--- a/src/yuzu/main.h
+++ b/src/yuzu/main.h
@@ -118,7 +118,7 @@ class GMainWindow : public QMainWindow {
 public:
     void filterBarSetChecked(bool state);
     void UpdateUITheme();
-    explicit GMainWindow();
+    explicit GMainWindow(bool has_broken_vulkan);
     ~GMainWindow() override;
 
     bool DropAction(QDropEvent* event);
diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
new file mode 100644
index 000000000..8421280bf
--- /dev/null
+++ b/src/yuzu/startup_checks.cpp
@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+#ifdef _WIN32
+#include <cstring> // for memset, strncpy
+#include <processthreadsapi.h>
+#include <windows.h>
+#elif defined(YUZU_UNIX)
+#include <errno.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#endif
+
+#include <cstdio>
+#include "video_core/vulkan_common/vulkan_instance.h"
+#include "video_core/vulkan_common/vulkan_library.h"
+#include "yuzu/startup_checks.h"
+
+void CheckVulkan() {
+    // Just start the Vulkan loader, this will crash if something is wrong
+    try {
+        Vulkan::vk::InstanceDispatch dld;
+        const Common::DynamicLibrary library = Vulkan::OpenLibrary();
+        const Vulkan::vk::Instance instance =
+            Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_0);
+
+    } catch (const Vulkan::vk::Exception& exception) {
+        std::fprintf(stderr, "Failed to initialize Vulkan: %s\n", exception.what());
+    }
+}
+
+bool StartupChecks(const char* arg0, bool* has_broken_vulkan) {
+#ifdef _WIN32
+    // Check environment variable to see if we are the child
+    char variable_contents[8];
+    const DWORD startup_check_var =
+        GetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, variable_contents, 8);
+    if (startup_check_var > 0 && std::strncmp(variable_contents, "ON", 8) == 0) {
+        CheckVulkan();
+        return true;
+    }
+
+    // Set the startup variable for child processes
+    const bool env_var_set = SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, "ON");
+    if (!env_var_set) {
+        std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %d\n",
+                     STARTUP_CHECK_ENV_VAR, GetLastError());
+        return false;
+    }
+
+    PROCESS_INFORMATION process_info;
+    std::memset(&process_info, '\0', sizeof(process_info));
+
+    if (!SpawnChild(arg0, &process_info)) {
+        return false;
+    }
+
+    // Wait until the processs exits and get exit code from it
+    WaitForSingleObject(process_info.hProcess, INFINITE);
+    DWORD exit_code = STILL_ACTIVE;
+    const int err = GetExitCodeProcess(process_info.hProcess, &exit_code);
+    if (err == 0) {
+        std::fprintf(stderr, "GetExitCodeProcess failed with error %d\n", GetLastError());
+    }
+
+    // Vulkan is broken if the child crashed (return value is not zero)
+    *has_broken_vulkan = (exit_code != 0);
+
+    if (CloseHandle(process_info.hProcess) == 0) {
+        std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError());
+    }
+    if (CloseHandle(process_info.hThread) == 0) {
+        std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError());
+    }
+
+    if (!SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, nullptr)) {
+        std::fprintf(stderr, "SetEnvironmentVariableA failed to clear %s with error %d\n",
+                     STARTUP_CHECK_ENV_VAR, GetLastError());
+    }
+
+#elif defined(YUZU_UNIX)
+    const pid_t pid = fork();
+    if (pid == 0) {
+        CheckVulkan();
+        return true;
+    } else if (pid == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "fork failed with error %d\n", err);
+        return false;
+    }
+
+    // Get exit code from child process
+    int status;
+    const int r_val = wait(&status);
+    if (r_val == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "wait failed with error %d\n", err);
+        return false;
+    }
+    // Vulkan is broken if the child crashed (return value is not zero)
+    *has_broken_vulkan = (status != 0);
+#endif
+    return false;
+}
+
+#ifdef _WIN32
+bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi) {
+    STARTUPINFOA startup_info;
+
+    std::memset(&startup_info, '\0', sizeof(startup_info));
+    startup_info.cb = sizeof(startup_info);
+
+    char p_name[255];
+    std::strncpy(p_name, arg0, 255);
+
+    const bool process_created = CreateProcessA(nullptr,       // lpApplicationName
+                                                p_name,        // lpCommandLine
+                                                nullptr,       // lpProcessAttributes
+                                                nullptr,       // lpThreadAttributes
+                                                false,         // bInheritHandles
+                                                0,             // dwCreationFlags
+                                                nullptr,       // lpEnvironment
+                                                nullptr,       // lpCurrentDirectory
+                                                &startup_info, // lpStartupInfo
+                                                pi             // lpProcessInformation
+    );
+    if (!process_created) {
+        std::fprintf(stderr, "CreateProcessA failed with error %d\n", GetLastError());
+        return false;
+    }
+
+    return true;
+}
+#endif
diff --git a/src/yuzu/startup_checks.h b/src/yuzu/startup_checks.h
new file mode 100644
index 000000000..096dd54a8
--- /dev/null
+++ b/src/yuzu/startup_checks.h
@@ -0,0 +1,17 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+constexpr char STARTUP_CHECK_ENV_VAR[] = "YUZU_DO_STARTUP_CHECKS";
+
+void CheckVulkan();
+bool StartupChecks(const char* arg0, bool* has_broken_vulkan);
+
+#ifdef _WIN32
+bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi);
+#endif
diff --git a/src/yuzu/uisettings.h b/src/yuzu/uisettings.h
index 044d88ca6..2f6948243 100644
--- a/src/yuzu/uisettings.h
+++ b/src/yuzu/uisettings.h
@@ -78,7 +78,7 @@ struct Values {
     Settings::Setting<bool> mute_when_in_background{false, "muteWhenInBackground"};
     Settings::Setting<bool> hide_mouse{true, "hideInactiveMouse"};
     // Set when Vulkan is known to crash the application
-    Settings::Setting<bool> has_broken_vulkan{false, "has_broken_vulkan"};
+    bool has_broken_vulkan = false;
 
     Settings::Setting<bool> select_user_on_boot{false, "select_user_on_boot"};