renderer_vulkan: Create debug callback on separate file and throw

Initialize debug callbacks (messenger) from a separate file. This allows
sharing code with different backends.

Change our Vulkan error handling to use exceptions instead of error
codes, simplifying the initialization process.

8 files changed, 88 insertions, 79 deletions

diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt
index f8c36947a..9287faee1 100644
--- a/src/video_core/CMakeLists.txt
+++ b/src/video_core/CMakeLists.txt
@@ -258,6 +258,8 @@ add_library(video_core STATIC
     textures/texture.h
     video_core.cpp
     video_core.h
+    vulkan_common/vulkan_debug_callback.cpp
+    vulkan_common/vulkan_debug_callback.h
     vulkan_common/vulkan_instance.cpp
     vulkan_common/vulkan_instance.h
     vulkan_common/vulkan_library.cpp
diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.cpp b/src/video_core/renderer_vulkan/renderer_vulkan.cpp
index 82619bc61..8e01dc191 100644
--- a/src/video_core/renderer_vulkan/renderer_vulkan.cpp
+++ b/src/video_core/renderer_vulkan/renderer_vulkan.cpp
@@ -29,6 +29,7 @@
 #include "video_core/renderer_vulkan/vk_scheduler.h"
 #include "video_core/renderer_vulkan/vk_state_tracker.h"
 #include "video_core/renderer_vulkan/vk_swapchain.h"
+#include "video_core/vulkan_common/vulkan_debug_callback.h"
 #include "video_core/vulkan_common/vulkan_instance.h"
 #include "video_core/vulkan_common/vulkan_library.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -48,24 +49,6 @@
 
 namespace Vulkan {
 namespace {
-VkBool32 DebugCallback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
-                       VkDebugUtilsMessageTypeFlagsEXT type,
-                       const VkDebugUtilsMessengerCallbackDataEXT* data,
-                       [[maybe_unused]] void* user_data) {
-    const char* const message{data->pMessage};
-
-    if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) {
-        LOG_CRITICAL(Render_Vulkan, "{}", message);
-    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) {
-        LOG_WARNING(Render_Vulkan, "{}", message);
-    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT) {
-        LOG_INFO(Render_Vulkan, "{}", message);
-    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT) {
-        LOG_DEBUG(Render_Vulkan, "{}", message);
-    }
-    return VK_FALSE;
-}
-
 std::string GetReadableVersion(u32 version) {
     return fmt::format("{}.{}.{}", VK_VERSION_MAJOR(version), VK_VERSION_MINOR(version),
                        VK_VERSION_PATCH(version));
@@ -158,7 +141,11 @@ bool RendererVulkan::Init() {
     library = OpenLibrary();
     std::tie(instance, instance_version) = CreateInstance(
         library, dld, render_window.GetWindowInfo().type, true, Settings::values.renderer_debug);
-    if (!instance || !CreateDebugCallback() || !CreateSurface() || !PickDevices()) {
+    if (Settings::values.renderer_debug) {
+        debug_callback = CreateDebugCallback(instance);
+    }
+
+    if (!CreateSurface() || !PickDevices()) {
         return false;
     }
 
@@ -201,18 +188,6 @@ void RendererVulkan::ShutDown() {
     device.reset();
 }
 
-bool RendererVulkan::CreateDebugCallback() {
-    if (!Settings::values.renderer_debug) {
-        return true;
-    }
-    debug_callback = instance.TryCreateDebugCallback(DebugCallback);
-    if (!debug_callback) {
-        LOG_ERROR(Render_Vulkan, "Failed to create debug callback");
-        return false;
-    }
-    return true;
-}
-
 bool RendererVulkan::CreateSurface() {
     [[maybe_unused]] const auto& window_info = render_window.GetWindowInfo();
     VkSurfaceKHR unsafe_surface = nullptr;
diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.h b/src/video_core/renderer_vulkan/renderer_vulkan.h
index 4a0abfaad..fa7628d0e 100644
--- a/src/video_core/renderer_vulkan/renderer_vulkan.h
+++ b/src/video_core/renderer_vulkan/renderer_vulkan.h
@@ -56,8 +56,6 @@ public:
     static std::vector<std::string> EnumerateDevices();
 
 private:
-    bool CreateDebugCallback();
-
     bool CreateSurface();
 
     bool PickDevices();
@@ -78,7 +76,7 @@ private:
 
     VKScreenInfo screen_info;
 
-    vk::DebugCallback debug_callback;
+    vk::DebugUtilsMessenger debug_callback;
     std::unique_ptr<VKDevice> device;
     std::unique_ptr<VKMemoryManager> memory_manager;
     std::unique_ptr<StateTracker> state_tracker;
diff --git a/src/video_core/vulkan_common/vulkan_debug_callback.cpp b/src/video_core/vulkan_common/vulkan_debug_callback.cpp
new file mode 100644
index 000000000..ea7af8ad4
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_debug_callback.cpp
@@ -0,0 +1,45 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <string_view>
+#include "common/logging/log.h"
+#include "video_core/vulkan_common/vulkan_debug_callback.h"
+
+namespace Vulkan {
+namespace {
+VkBool32 Callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
+                  VkDebugUtilsMessageTypeFlagsEXT type,
+                  const VkDebugUtilsMessengerCallbackDataEXT* data,
+                  [[maybe_unused]] void* user_data) {
+    const std::string_view message{data->pMessage};
+    if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) {
+        LOG_CRITICAL(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) {
+        LOG_WARNING(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT) {
+        LOG_INFO(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT) {
+        LOG_DEBUG(Render_Vulkan, "{}", message);
+    }
+    return VK_FALSE;
+}
+} // Anonymous namespace
+
+vk::DebugUtilsMessenger CreateDebugCallback(const vk::Instance& instance) {
+    return instance.CreateDebugUtilsMessenger(VkDebugUtilsMessengerCreateInfoEXT{
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
+        .pNext = nullptr,
+        .flags = 0,
+        .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT,
+        .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
+                       VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
+                       VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
+        .pfnUserCallback = Callback,
+    });
+}
+
+} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_debug_callback.h b/src/video_core/vulkan_common/vulkan_debug_callback.h
new file mode 100644
index 000000000..2efcd244c
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_debug_callback.h
@@ -0,0 +1,11 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+namespace Vulkan {
+
+vk::DebugUtilsMessenger CreateDebugCallback(const vk::Instance& instance);
+
+} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_instance.cpp b/src/video_core/vulkan_common/vulkan_instance.cpp
index c19f93e0a..d3d8630e5 100644
--- a/src/video_core/vulkan_common/vulkan_instance.cpp
+++ b/src/video_core/vulkan_common/vulkan_instance.cpp
@@ -117,21 +117,20 @@ std::pair<vk::Instance, u32> CreateInstance(Common::DynamicLibrary& library,
                                             bool enable_debug_utils, bool enable_layers) {
     if (!library.IsOpen()) {
         LOG_ERROR(Render_Vulkan, "Vulkan library not available");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     if (!library.GetSymbol("vkGetInstanceProcAddr", &dld.vkGetInstanceProcAddr)) {
         LOG_ERROR(Render_Vulkan, "vkGetInstanceProcAddr not present in Vulkan");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     if (!vk::Load(dld)) {
         LOG_ERROR(Render_Vulkan, "Failed to load Vulkan function pointers");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     const std::vector<const char*> extensions = RequiredExtensions(window_type, enable_debug_utils);
     if (!AreExtensionsSupported(dld, extensions)) {
-        return {};
+        throw vk::Exception(VK_ERROR_EXTENSION_NOT_PRESENT);
     }
-
     std::vector<const char*> layers = Layers(enable_layers);
     RemoveUnavailableLayers(dld, layers);
 
@@ -139,12 +138,9 @@ std::pair<vk::Instance, u32> CreateInstance(Common::DynamicLibrary& library,
     const u32 version = std::min(vk::AvailableVersion(dld), VK_API_VERSION_1_1);
 
     vk::Instance instance = vk::Instance::Create(version, layers, extensions, dld);
-    if (!instance) {
-        LOG_ERROR(Render_Vulkan, "Failed to create Vulkan instance");
-        return {};
-    }
     if (!vk::Load(*instance, dld)) {
         LOG_ERROR(Render_Vulkan, "Failed to load Vulkan instance function pointers");
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     return std::make_pair(std::move(instance), version);
 }
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp
index 478402bbd..f4177537b 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.cpp
+++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp
@@ -435,7 +435,7 @@ VkResult Free(VkDevice device, VkCommandPool handle, Span<VkCommandBuffer> buffe
 }
 
 Instance Instance::Create(u32 version, Span<const char*> layers, Span<const char*> extensions,
-                          InstanceDispatch& dispatch) noexcept {
+                          InstanceDispatch& dispatch) {
     const VkApplicationInfo application_info{
         .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
         .pNext = nullptr,
@@ -455,22 +455,17 @@ Instance Instance::Create(u32 version, Span<const char*> layers, Span<const char
         .enabledExtensionCount = extensions.size(),
         .ppEnabledExtensionNames = extensions.data(),
     };
-
     VkInstance instance;
-    if (dispatch.vkCreateInstance(&ci, nullptr, &instance) != VK_SUCCESS) {
-        // Failed to create the instance.
-        return {};
-    }
+    Check(dispatch.vkCreateInstance(&ci, nullptr, &instance));
     if (!Proc(dispatch.vkDestroyInstance, dispatch, "vkDestroyInstance", instance)) {
         // We successfully created an instance but the destroy function couldn't be loaded.
         // This is a good moment to panic.
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
-
     return Instance(instance, dispatch);
 }
 
-std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices() {
+std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices() const {
     u32 num;
     if (dld->vkEnumeratePhysicalDevices(handle, &num, nullptr) != VK_SUCCESS) {
         return std::nullopt;
@@ -483,27 +478,11 @@ std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices(
     return std::make_optional(std::move(physical_devices));
 }
 
-DebugCallback Instance::TryCreateDebugCallback(
-    PFN_vkDebugUtilsMessengerCallbackEXT callback) noexcept {
-    const VkDebugUtilsMessengerCreateInfoEXT ci{
-        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
-        .pNext = nullptr,
-        .flags = 0,
-        .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT,
-        .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
-                       VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT,
-        .pfnUserCallback = callback,
-        .pUserData = nullptr,
-    };
-
-    VkDebugUtilsMessengerEXT messenger;
-    if (dld->vkCreateDebugUtilsMessengerEXT(handle, &ci, nullptr, &messenger) != VK_SUCCESS) {
-        return {};
-    }
-    return DebugCallback(messenger, handle, *dld);
+DebugUtilsMessenger Instance::CreateDebugUtilsMessenger(
+    const VkDebugUtilsMessengerCreateInfoEXT& create_info) const {
+    VkDebugUtilsMessengerEXT object;
+    Check(dld->vkCreateDebugUtilsMessengerEXT(handle, &create_info, nullptr, &object));
+    return DebugUtilsMessenger(object, handle, *dld);
 }
 
 void Buffer::BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const {
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index f9a184e00..03ca97ac0 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -555,7 +555,7 @@ private:
     const DeviceDispatch* dld = nullptr;
 };
 
-using DebugCallback = Handle<VkDebugUtilsMessengerEXT, VkInstance, InstanceDispatch>;
+using DebugUtilsMessenger = Handle<VkDebugUtilsMessengerEXT, VkInstance, InstanceDispatch>;
 using DescriptorSetLayout = Handle<VkDescriptorSetLayout, VkDevice, DeviceDispatch>;
 using DescriptorUpdateTemplateKHR = Handle<VkDescriptorUpdateTemplateKHR, VkDevice, DeviceDispatch>;
 using Pipeline = Handle<VkPipeline, VkDevice, DeviceDispatch>;
@@ -573,16 +573,19 @@ class Instance : public Handle<VkInstance, NoOwner, InstanceDispatch> {
     using Handle<VkInstance, NoOwner, InstanceDispatch>::Handle;
 
 public:
-    /// Creates a Vulkan instance. Use "operator bool" for error handling.
+    /// Creates a Vulkan instance.
+    /// @throw Exception on initialization error.
     static Instance Create(u32 version, Span<const char*> layers, Span<const char*> extensions,
-                           InstanceDispatch& dispatch) noexcept;
+                           InstanceDispatch& dispatch);
 
     /// Enumerates physical devices.
     /// @return Physical devices and an empty handle on failure.
-    std::optional<std::vector<VkPhysicalDevice>> EnumeratePhysicalDevices();
+    std::optional<std::vector<VkPhysicalDevice>> EnumeratePhysicalDevices() const;
 
-    /// Tries to create a debug callback messenger. Returns an empty handle on failure.
-    DebugCallback TryCreateDebugCallback(PFN_vkDebugUtilsMessengerCallbackEXT callback) noexcept;
+    /// Creates a debug callback messenger.
+    /// @throw Exception on creation failure.
+    DebugUtilsMessenger CreateDebugUtilsMessenger(
+        const VkDebugUtilsMessengerCreateInfoEXT& create_info) const;
 };
 
 class Queue {