yuzu-qt: Attempt to workaround broken Vulkan installations

This does a few things in order to make the default setting Vulkan
workable.

- When yuzu boots, it just opens the Vulkan library.
  - If it works, all good and we continue with Vulkan as the default.
  - If something breaks, a new file in the config directory will be left
    behind (this is deleted normally).
- If Vulkan is not working, has_broken_vulkan is set to true.
  - The first time this happens, a warning is displayed to notify the
    user.
  - This forces use of OpenGL, and Vulkan cannot be selected.
  - The Shader Backend selector is made accessible for use in custom
    configurations.
  - To disable has_broken_vulkan, the user needs to press a button in
    Graphics Configuration to manually run the Vulkan device
    enumeration.

9 files changed, 166 insertions, 46 deletions

diff --git a/src/yuzu/CMakeLists.txt b/src/yuzu/CMakeLists.txt
index 404acdd05..39989885d 100644
--- a/src/yuzu/CMakeLists.txt
+++ b/src/yuzu/CMakeLists.txt
@@ -30,6 +30,8 @@ add_executable(yuzu
     applets/qt_web_browser_scripts.h
     bootmanager.cpp
     bootmanager.h
+    check_vulkan.cpp
+    check_vulkan.h
     compatdb.ui
     compatibility_list.cpp
     compatibility_list.h
diff --git a/src/yuzu/check_vulkan.cpp b/src/yuzu/check_vulkan.cpp
new file mode 100644
index 000000000..1b21efe69
--- /dev/null
+++ b/src/yuzu/check_vulkan.cpp
@@ -0,0 +1,51 @@
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+#include <exception>
+#include <filesystem>
+#include <fstream>
+#include "common/fs/fs.h"
+#include "common/fs/path_util.h"
+#include "common/logging/log.h"
+#include "video_core/vulkan_common/vulkan_instance.h"
+#include "video_core/vulkan_common/vulkan_library.h"
+#include "yuzu/check_vulkan.h"
+#include "yuzu/uisettings.h"
+
+constexpr char TEMP_FILE_NAME[] = "vulkan_check";
+
+bool CheckVulkan() {
+    if (UISettings::values.has_broken_vulkan) {
+        return true;
+    }
+
+    LOG_DEBUG(Frontend, "Checking presence of Vulkan");
+
+    const auto fs_config_loc = Common::FS::GetYuzuPath(Common::FS::YuzuPath::ConfigDir);
+    const auto temp_file_loc = fs_config_loc / TEMP_FILE_NAME;
+
+    if (std::filesystem::exists(temp_file_loc)) {
+        LOG_WARNING(Frontend, "Detected recovery from previous failed Vulkan initialization");
+
+        UISettings::values.has_broken_vulkan = true;
+        std::filesystem::remove(temp_file_loc);
+        return false;
+    }
+
+    std::ofstream temp_file_handle(temp_file_loc);
+    temp_file_handle.close();
+
+    try {
+        Vulkan::vk::InstanceDispatch dld;
+        const Common::DynamicLibrary library = Vulkan::OpenLibrary();
+        const Vulkan::vk::Instance instance =
+            Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_0);
+
+    } catch (const Vulkan::vk::Exception& exception) {
+        LOG_ERROR(Frontend, "Failed to initialize Vulkan: {}", exception.what());
+        UISettings::values.has_broken_vulkan = true;
+        return false;
+    }
+
+    std::filesystem::remove(temp_file_loc);
+    return true;
+}
diff --git a/src/yuzu/check_vulkan.h b/src/yuzu/check_vulkan.h
new file mode 100644
index 000000000..3b199d3bb
--- /dev/null
+++ b/src/yuzu/check_vulkan.h
@@ -0,0 +1 @@
+bool CheckVulkan();
diff --git a/src/yuzu/configuration/config.cpp b/src/yuzu/configuration/config.cpp
index ac26b885b..8b95b677b 100644
--- a/src/yuzu/configuration/config.cpp
+++ b/src/yuzu/configuration/config.cpp
@@ -679,6 +679,12 @@ void Config::ReadRendererValues() {
     ReadGlobalSetting(Settings::values.bg_green);
     ReadGlobalSetting(Settings::values.bg_blue);
 
+    if (!global && UISettings::values.has_broken_vulkan &&
+        Settings::values.renderer_backend.GetValue() == Settings::RendererBackend::Vulkan &&
+        !Settings::values.renderer_backend.UsingGlobal()) {
+        Settings::values.renderer_backend.SetGlobal(true);
+    }
+
     if (global) {
         ReadBasicSetting(Settings::values.renderer_debug);
         ReadBasicSetting(Settings::values.renderer_shader_feedback);
@@ -798,6 +804,7 @@ void Config::ReadUIValues() {
     ReadBasicSetting(UISettings::values.pause_when_in_background);
     ReadBasicSetting(UISettings::values.mute_when_in_background);
     ReadBasicSetting(UISettings::values.hide_mouse);
+    ReadBasicSetting(UISettings::values.has_broken_vulkan);
     ReadBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
@@ -1343,6 +1350,7 @@ void Config::SaveUIValues() {
     WriteBasicSetting(UISettings::values.pause_when_in_background);
     WriteBasicSetting(UISettings::values.mute_when_in_background);
     WriteBasicSetting(UISettings::values.hide_mouse);
+    WriteBasicSetting(UISettings::values.has_broken_vulkan);
     WriteBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp
index 2f1435b10..482a6a8ab 100644
--- a/src/yuzu/configuration/configure_graphics.cpp
+++ b/src/yuzu/configuration/configure_graphics.cpp
@@ -5,6 +5,7 @@
 // Include this early to include Vulkan headers how we want to
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
+#include <exception>
 #include <QColorDialog>
 #include <QVulkanInstance>
 
@@ -17,6 +18,7 @@
 #include "video_core/vulkan_common/vulkan_library.h"
 #include "yuzu/configuration/configuration_shared.h"
 #include "yuzu/configuration/configure_graphics.h"
+#include "yuzu/uisettings.h"
 
 ConfigureGraphics::ConfigureGraphics(const Core::System& system_, QWidget* parent)
     : QWidget(parent), ui{std::make_unique<Ui::ConfigureGraphics>()}, system{system_} {
@@ -57,6 +59,23 @@ ConfigureGraphics::ConfigureGraphics(const Core::System& system_, QWidget* paren
         UpdateBackgroundColorButton(new_bg_color);
     });
 
+    connect(ui->button_check_vulkan, &QAbstractButton::clicked, this, [this] {
+        UISettings::values.has_broken_vulkan = false;
+
+        if (RetrieveVulkanDevices()) {
+            ui->api->setEnabled(true);
+
+            for (const auto& device : vulkan_devices) {
+                ui->device->addItem(device);
+            }
+        } else {
+            UISettings::values.has_broken_vulkan = true;
+        }
+    });
+
+    ui->api->setEnabled(!UISettings::values.has_broken_vulkan.GetValue());
+    ui->button_check_vulkan->setVisible(UISettings::values.has_broken_vulkan.GetValue());
+
     ui->bg_label->setVisible(Settings::IsConfiguringGlobal());
     ui->bg_combobox->setVisible(!Settings::IsConfiguringGlobal());
 }
@@ -296,7 +315,7 @@ void ConfigureGraphics::UpdateAPILayout() {
         vulkan_device = Settings::values.vulkan_device.GetValue(true);
         shader_backend = Settings::values.shader_backend.GetValue(true);
         ui->device_widget->setEnabled(false);
-        ui->backend_widget->setEnabled(false);
+        ui->backend_widget->setEnabled(UISettings::values.has_broken_vulkan.GetValue());
     } else {
         vulkan_device = Settings::values.vulkan_device.GetValue();
         shader_backend = Settings::values.shader_backend.GetValue();
@@ -318,7 +337,11 @@ void ConfigureGraphics::UpdateAPILayout() {
     }
 }
 
-void ConfigureGraphics::RetrieveVulkanDevices() try {
+bool ConfigureGraphics::RetrieveVulkanDevices() try {
+    if (UISettings::values.has_broken_vulkan) {
+        return false;
+    }
+
     using namespace Vulkan;
 
     vk::InstanceDispatch dld;
@@ -333,8 +356,13 @@ void ConfigureGraphics::RetrieveVulkanDevices() try {
         vulkan_devices.push_back(QString::fromStdString(name));
     }
 
+    UISettings::values.has_broken_vulkan = false;
+    ui->button_check_vulkan->setVisible(false);
+
+    return true;
 } catch (const Vulkan::vk::Exception& exception) {
     LOG_ERROR(Frontend, "Failed to enumerate devices with error: {}", exception.what());
+    return false;
 }
 
 Settings::RendererBackend ConfigureGraphics::GetCurrentGraphicsBackend() const {
@@ -415,4 +443,11 @@ void ConfigureGraphics::SetupPerGameUI() {
         ui->api, static_cast<int>(Settings::values.renderer_backend.GetValue(true)));
     ConfigurationShared::InsertGlobalItem(
         ui->nvdec_emulation, static_cast<int>(Settings::values.nvdec_emulation.GetValue(true)));
+
+    if (UISettings::values.has_broken_vulkan) {
+        ui->backend_widget->setEnabled(true);
+        ConfigurationShared::SetColoredComboBox(
+            ui->backend, ui->backend_widget,
+            static_cast<int>(Settings::values.shader_backend.GetValue(true)));
+    }
 }
diff --git a/src/yuzu/configuration/configure_graphics.h b/src/yuzu/configuration/configure_graphics.h
index 1b101c940..8438f0187 100644
--- a/src/yuzu/configuration/configure_graphics.h
+++ b/src/yuzu/configuration/configure_graphics.h
@@ -41,7 +41,7 @@ private:
     void UpdateDeviceSelection(int device);
     void UpdateShaderBackendSelection(int backend);
 
-    void RetrieveVulkanDevices();
+    bool RetrieveVulkanDevices();
 
     void SetupPerGameUI();
 
diff --git a/src/yuzu/configuration/configure_graphics.ui b/src/yuzu/configuration/configure_graphics.ui
index 74f0e0b79..2f94c94bc 100644
--- a/src/yuzu/configuration/configure_graphics.ui
+++ b/src/yuzu/configuration/configure_graphics.ui
@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>437</width>
-    <height>482</height>
+    <width>471</width>
+    <height>759</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -171,11 +171,11 @@
          </widget>
         </item>
         <item>
-          <widget class="QCheckBox" name="accelerate_astc">
-            <property name="text">
-              <string>Accelerate ASTC texture decoding</string>
-            </property>
-          </widget>
+         <widget class="QCheckBox" name="accelerate_astc">
+          <property name="text">
+           <string>Accelerate ASTC texture decoding</string>
+          </property>
+         </widget>
         </item>
         <item>
          <widget class="QWidget" name="nvdec_emulation_widget" native="true">
@@ -438,43 +438,43 @@
          </widget>
         </item>
         <item>
-          <widget class="QWidget" name="anti_aliasing_layout" native="true">
-            <layout class="QHBoxLayout" name="horizontalLayout_7">
-              <property name="leftMargin">
-                <number>0</number>
-              </property>
-              <property name="topMargin">
-                <number>0</number>
-              </property>
-              <property name="rightMargin">
-                <number>0</number>
+         <widget class="QWidget" name="anti_aliasing_layout" native="true">
+          <layout class="QHBoxLayout" name="horizontalLayout_7">
+           <property name="leftMargin">
+            <number>0</number>
+           </property>
+           <property name="topMargin">
+            <number>0</number>
+           </property>
+           <property name="rightMargin">
+            <number>0</number>
+           </property>
+           <property name="bottomMargin">
+            <number>0</number>
+           </property>
+           <item>
+            <widget class="QLabel" name="anti_aliasing_label">
+             <property name="text">
+              <string>Anti-Aliasing Method:</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QComboBox" name="anti_aliasing_combobox">
+             <item>
+              <property name="text">
+               <string>None</string>
               </property>
-              <property name="bottomMargin">
-                <number>0</number>
+             </item>
+             <item>
+              <property name="text">
+               <string>FXAA</string>
               </property>
-              <item>
-                <widget class="QLabel" name="anti_aliasing_label">
-                  <property name="text">
-                    <string>Anti-Aliasing Method:</string>
-                  </property>
-                </widget>
-              </item>
-              <item>
-                <widget class="QComboBox" name="anti_aliasing_combobox">
-                  <item>
-                    <property name="text">
-                      <string>None</string>
-                    </property>
-                  </item>
-                  <item>
-                    <property name="text">
-                      <string>FXAA</string>
-                    </property>
-                  </item>
-                </widget>
-              </item>
-            </layout>
-          </widget>
+             </item>
+            </widget>
+           </item>
+          </layout>
+         </widget>
         </item>
         <item>
          <widget class="QWidget" name="bg_layout" native="true">
@@ -574,6 +574,13 @@
      </property>
     </spacer>
    </item>
+   <item>
+    <widget class="QPushButton" name="button_check_vulkan">
+     <property name="text">
+      <string>Check for Working Vulkan</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources/>
diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp
index f4a9a7171..71802cfc2 100644
--- a/src/yuzu/main.cpp
+++ b/src/yuzu/main.cpp
@@ -115,6 +115,7 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual
 #include "video_core/shader_notify.h"
 #include "yuzu/about_dialog.h"
 #include "yuzu/bootmanager.h"
+#include "yuzu/check_vulkan.h"
 #include "yuzu/compatdb.h"
 #include "yuzu/compatibility_list.h"
 #include "yuzu/configuration/config.h"
@@ -297,6 +298,20 @@ GMainWindow::GMainWindow()
 
     MigrateConfigFiles();
 
+    if (!CheckVulkan()) {
+        QMessageBox::warning(
+            this, tr("Broken Vulkan Installation Detected"),
+            tr("Vulkan initialization failed on the previous boot. Please update your graphics "
+               "driver, then re-check your Vulkan installation by accessing the Graphics "
+               "configuration and clicking \"Check for Working Vulkan\"."));
+    }
+    if (UISettings::values.has_broken_vulkan) {
+        Settings::values.renderer_backend = Settings::RendererBackend::OpenGL;
+
+        renderer_status_button->setDisabled(true);
+        renderer_status_button->setChecked(false);
+    }
+
 #if defined(HAVE_SDL2) && !defined(_WIN32)
     SDL_InitSubSystem(SDL_INIT_VIDEO);
     // SDL disables the screen saver by default, and setting the hint
@@ -1563,7 +1578,7 @@ void GMainWindow::ShutdownGame() {
     emu_speed_label->setVisible(false);
     game_fps_label->setVisible(false);
     emu_frametime_label->setVisible(false);
-    renderer_status_button->setEnabled(true);
+    renderer_status_button->setEnabled(!UISettings::values.has_broken_vulkan);
 
     game_path.clear();
 
diff --git a/src/yuzu/uisettings.h b/src/yuzu/uisettings.h
index 15ba9ea17..653b76883 100644
--- a/src/yuzu/uisettings.h
+++ b/src/yuzu/uisettings.h
@@ -77,6 +77,7 @@ struct Values {
     Settings::BasicSetting<bool> pause_when_in_background{false, "pauseWhenInBackground"};
     Settings::BasicSetting<bool> mute_when_in_background{false, "muteWhenInBackground"};
     Settings::BasicSetting<bool> hide_mouse{true, "hideInactiveMouse"};
+    Settings::BasicSetting<bool> has_broken_vulkan{false, "has_broken_vulkan"};
 
     Settings::BasicSetting<bool> select_user_on_boot{false, "select_user_on_boot"};