renderer_vulkan: Move instance initialization to a separate file

Simplify Vulkan's backend initialization code by moving it to a separate
file, allowing us to initialize a Vulkan instance from different
backends.

4 files changed, 176 insertions, 111 deletions

diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index e19632bb1..f8c36947a 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -258,6 +258,8 @@ add_library(video_core STATIC
     textures/texture.h
     video_core.cpp
     video_core.h
+    vulkan_common/vulkan_instance.cpp
+    vulkan_common/vulkan_instance.h
     vulkan_common/vulkan_library.cpp
     vulkan_common/vulkan_library.h
     vulkan_common/vulkan_wrapper.cpp
diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.cpp b/src/video_core/renderer_vulkan/renderer_vulkan.cpp
index 6e267f89d..82619bc61 100644
--- a/src/video_core/renderer_vulkan/renderer_vulkan.cpp
+++ b/src/video_core/renderer_vulkan/renderer_vulkan.cpp
@@ -29,6 +29,7 @@
 #include "video_core/renderer_vulkan/vk_scheduler.h"
 #include "video_core/renderer_vulkan/vk_state_tracker.h"
 #include "video_core/renderer_vulkan/vk_swapchain.h"
+#include "video_core/vulkan_common/vulkan_instance.h"
 #include "video_core/vulkan_common/vulkan_library.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
@@ -46,11 +47,7 @@
 #endif
 
 namespace Vulkan {
-
 namespace {
-
-using Core::Frontend::WindowSystemType;
-
 VkBool32 DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
                        VkDebugUtilsMessageTypeFlagsEXT type,
                        const VkDebugUtilsMessengerCallbackDataEXT* data,
@@ -69,109 +66,6 @@ VkBool32 DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
     return VK_FALSE;
 }
 
-std::pair<vk::Instance, u32> CreateInstance(
-    Common::DynamicLibrary& library, vk::InstanceDispatch& dld,
-    WindowSystemType window_type = WindowSystemType::Headless, bool enable_debug_utils = false,
-    bool enable_layers = false) {
-    if (!library.IsOpen()) {
-        LOG_ERROR(Render_Vulkan, "Vulkan library not available");
-        return {};
-    }
-    if (!library.GetSymbol("vkGetInstanceProcAddr", &dld.vkGetInstanceProcAddr)) {
-        LOG_ERROR(Render_Vulkan, "vkGetInstanceProcAddr not present in Vulkan");
-        return {};
-    }
-    if (!vk::Load(dld)) {
-        LOG_ERROR(Render_Vulkan, "Failed to load Vulkan function pointers");
-        return {};
-    }
-
-    std::vector<const char*> extensions;
-    extensions.reserve(6);
-    switch (window_type) {
-    case Core::Frontend::WindowSystemType::Headless:
-        break;
-#ifdef _WIN32
-    case Core::Frontend::WindowSystemType::Windows:
-        extensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
-        break;
-#endif
-#if !defined(_WIN32) && !defined(__APPLE__)
-    case Core::Frontend::WindowSystemType::X11:
-        extensions.push_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
-        break;
-    case Core::Frontend::WindowSystemType::Wayland:
-        extensions.push_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
-        break;
-#endif
-    default:
-        LOG_ERROR(Render_Vulkan, "Presentation not supported on this platform");
-        break;
-    }
-    if (window_type != Core::Frontend::WindowSystemType::Headless) {
-        extensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
-    }
-    if (enable_debug_utils) {
-        extensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
-    }
-    extensions.push_back(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME);
-
-    const std::optional properties = vk::EnumerateInstanceExtensionProperties(dld);
-    if (!properties) {
-        LOG_ERROR(Render_Vulkan, "Failed to query extension properties");
-        return {};
-    }
-
-    for (const char* extension : extensions) {
-        const auto it =
-            std::find_if(properties->begin(), properties->end(), [extension](const auto& prop) {
-                return !std::strcmp(extension, prop.extensionName);
-            });
-        if (it == properties->end()) {
-            LOG_ERROR(Render_Vulkan, "Required instance extension {} is not available", extension);
-            return {};
-        }
-    }
-
-    std::vector<const char*> layers;
-    layers.reserve(1);
-    if (enable_layers) {
-        layers.push_back("VK_LAYER_KHRONOS_validation");
-    }
-
-    const std::optional layer_properties = vk::EnumerateInstanceLayerProperties(dld);
-    if (!layer_properties) {
-        LOG_ERROR(Render_Vulkan, "Failed to query layer properties, disabling layers");
-        layers.clear();
-    }
-
-    for (auto layer_it = layers.begin(); layer_it != layers.end();) {
-        const char* const layer = *layer_it;
-        const auto it = std::find_if(
-            layer_properties->begin(), layer_properties->end(),
-            [layer](const VkLayerProperties& prop) { return !std::strcmp(layer, prop.layerName); });
-        if (it == layer_properties->end()) {
-            LOG_ERROR(Render_Vulkan, "Layer {} not available, removing it", layer);
-            layer_it = layers.erase(layer_it);
-        } else {
-            ++layer_it;
-        }
-    }
-
-    // Limit the maximum version of Vulkan to avoid using untested version.
-    const u32 version = std::min(vk::AvailableVersion(dld), static_cast<u32>(VK_API_VERSION_1_1));
-
-    vk::Instance instance = vk::Instance::Create(version, layers, extensions, dld);
-    if (!instance) {
-        LOG_ERROR(Render_Vulkan, "Failed to create Vulkan instance");
-        return {};
-    }
-    if (!vk::Load(*instance, dld)) {
-        LOG_ERROR(Render_Vulkan, "Failed to load Vulkan instance function pointers");
-    }
-    return std::make_pair(std::move(instance), version);
-}
-
 std::string GetReadableVersion(u32 version) {
     return fmt::format("{}.{}.{}", VK_VERSION_MAJOR(version), VK_VERSION_MINOR(version),
                        VK_VERSION_PATCH(version));
@@ -194,7 +88,6 @@ std::string GetDriverVersion(const VKDevice& device) {
         const u32 minor = version & 0x3fff;
         return fmt::format("{}.{}", major, minor);
     }
-
     return GetReadableVersion(version);
 }
 
@@ -233,7 +126,6 @@ void RendererVulkan::SwapBuffers(const Tegra::FramebufferConfig* framebuffer) {
     if (!framebuffer) {
         return;
     }
-
     const auto& layout = render_window.GetFramebufferLayout();
     if (layout.width > 0 && layout.height > 0 && render_window.IsShown()) {
         const VAddr framebuffer_addr = framebuffer->address + framebuffer->offset;
@@ -429,12 +321,10 @@ std::vector<std::string> RendererVulkan::EnumerateDevices() {
     if (!instance) {
         return {};
     }
-
     const std::optional physical_devices = instance.EnumeratePhysicalDevices();
     if (!physical_devices) {
         return {};
     }
-
     std::vector<std::string> names;
     names.reserve(physical_devices->size());
     for (const auto& device : *physical_devices) {
diff --git a/src/video_core/vulkan_common/vulkan_instance.cpp b/src/video_core/vulkan_common/vulkan_instance.cpp
new file mode 100644
index 000000000..c19f93e0a
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_instance.cpp
@@ -0,0 +1,152 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <algorithm>
+#include <optional>
+#include <span>
+#include <utility>
+#include <vector>
+
+#include "common/common_types.h"
+#include "common/dynamic_library.h"
+#include "common/logging/log.h"
+#include "core/frontend/emu_window.h"
+#include "video_core/vulkan_common/vulkan_instance.h"
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+// Include these late to avoid polluting previous headers
+#ifdef _WIN32
+#include <windows.h>
+// ensure include order
+#include <vulkan/vulkan_win32.h>
+#endif
+
+#if !defined(_WIN32) && !defined(__APPLE__)
+#include <X11/Xlib.h>
+#include <vulkan/vulkan_wayland.h>
+#include <vulkan/vulkan_xlib.h>
+#endif
+
+namespace Vulkan {
+namespace {
+[[nodiscard]] std::vector<const char*> RequiredExtensions(
+    Core::Frontend::WindowSystemType window_type, bool enable_debug_utils) {
+    std::vector<const char*> extensions;
+    extensions.reserve(6);
+    switch (window_type) {
+    case Core::Frontend::WindowSystemType::Headless:
+        break;
+#ifdef _WIN32
+    case Core::Frontend::WindowSystemType::Windows:
+        extensions.push_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
+        break;
+#endif
+#if !defined(_WIN32) && !defined(__APPLE__)
+    case Core::Frontend::WindowSystemType::X11:
+        extensions.push_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+        break;
+    case Core::Frontend::WindowSystemType::Wayland:
+        extensions.push_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
+        break;
+#endif
+    default:
+        LOG_ERROR(Render_Vulkan, "Presentation not supported on this platform");
+        break;
+    }
+    if (window_type != Core::Frontend::WindowSystemType::Headless) {
+        extensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
+    }
+    if (enable_debug_utils) {
+        extensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
+    }
+    extensions.push_back(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME);
+    return extensions;
+}
+
+[[nodiscard]] bool AreExtensionsSupported(const vk::InstanceDispatch& dld,
+                                          std::span<const char* const> extensions) {
+    const std::optional properties = vk::EnumerateInstanceExtensionProperties(dld);
+    if (!properties) {
+        LOG_ERROR(Render_Vulkan, "Failed to query extension properties");
+        return false;
+    }
+    for (const char* extension : extensions) {
+        const auto it = std::ranges::find_if(*properties, [extension](const auto& prop) {
+            return std::strcmp(extension, prop.extensionName) == 0;
+        });
+        if (it == properties->end()) {
+            LOG_ERROR(Render_Vulkan, "Required instance extension {} is not available", extension);
+            return false;
+        }
+    }
+    return true;
+}
+
+[[nodiscard]] std::vector<const char*> Layers(bool enable_layers) {
+    std::vector<const char*> layers;
+    if (enable_layers) {
+        layers.push_back("VK_LAYER_KHRONOS_validation");
+    }
+    return layers;
+}
+
+void RemoveUnavailableLayers(const vk::InstanceDispatch& dld, std::vector<const char*>& layers) {
+    const std::optional layer_properties = vk::EnumerateInstanceLayerProperties(dld);
+    if (!layer_properties) {
+        LOG_ERROR(Render_Vulkan, "Failed to query layer properties, disabling layers");
+        layers.clear();
+    }
+    std::erase_if(layers, [&layer_properties](const char* layer) {
+        const auto comp = [layer](const VkLayerProperties& layer_property) {
+            return std::strcmp(layer, layer_property.layerName) == 0;
+        };
+        const auto it = std::ranges::find_if(*layer_properties, comp);
+        if (it == layer_properties->end()) {
+            LOG_ERROR(Render_Vulkan, "Layer {} not available, removing it", layer);
+            return true;
+        }
+        return false;
+    });
+}
+} // Anonymous namespace
+
+std::pair<vk::Instance, u32> CreateInstance(Common::DynamicLibrary& library,
+                                            vk::InstanceDispatch& dld,
+                                            Core::Frontend::WindowSystemType window_type,
+                                            bool enable_debug_utils, bool enable_layers) {
+    if (!library.IsOpen()) {
+        LOG_ERROR(Render_Vulkan, "Vulkan library not available");
+        return {};
+    }
+    if (!library.GetSymbol("vkGetInstanceProcAddr", &dld.vkGetInstanceProcAddr)) {
+        LOG_ERROR(Render_Vulkan, "vkGetInstanceProcAddr not present in Vulkan");
+        return {};
+    }
+    if (!vk::Load(dld)) {
+        LOG_ERROR(Render_Vulkan, "Failed to load Vulkan function pointers");
+        return {};
+    }
+    const std::vector<const char*> extensions = RequiredExtensions(window_type, enable_debug_utils);
+    if (!AreExtensionsSupported(dld, extensions)) {
+        return {};
+    }
+
+    std::vector<const char*> layers = Layers(enable_layers);
+    RemoveUnavailableLayers(dld, layers);
+
+    // Limit the maximum version of Vulkan to avoid using untested version.
+    const u32 version = std::min(vk::AvailableVersion(dld), VK_API_VERSION_1_1);
+
+    vk::Instance instance = vk::Instance::Create(version, layers, extensions, dld);
+    if (!instance) {
+        LOG_ERROR(Render_Vulkan, "Failed to create Vulkan instance");
+        return {};
+    }
+    if (!vk::Load(*instance, dld)) {
+        LOG_ERROR(Render_Vulkan, "Failed to load Vulkan instance function pointers");
+    }
+    return std::make_pair(std::move(instance), version);
+}
+
+} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_instance.h b/src/video_core/vulkan_common/vulkan_instance.h
new file mode 100644
index 000000000..ff2be0a48
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_instance.h
@@ -0,0 +1,21 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <utility>
+
+#include "common/common_types.h"
+#include "common/dynamic_library.h"
+#include "core/frontend/emu_window.h"
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+namespace Vulkan {
+
+[[nodiscard]] std::pair<vk::Instance, u32> CreateInstance(
+    Common::DynamicLibrary& library, vk::InstanceDispatch& dld,
+    Core::Frontend::WindowSystemType window_type = Core::Frontend::WindowSystemType::Headless,
+    bool enable_debug_utils = false, bool enable_layers = false);
+
+} // namespace Vulkan