renderer_vulkan: Create debug callback on separate file and throw

Initialize debug callbacks (messenger) from a separate file. This allows
sharing code with different backends.

Change our Vulkan error handling to use exceptions instead of error
codes, simplifying the initialization process.

5 files changed, 79 insertions, 45 deletions

diff --git a/src/video_core/vulkan_common/vulkan_debug_callback.cpp b/src/video_core/vulkan_common/vulkan_debug_callback.cpp
new file mode 100644
index 000000000..ea7af8ad4
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_debug_callback.cpp
@@ -0,0 +1,45 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <string_view>
+#include "common/logging/log.h"
+#include "video_core/vulkan_common/vulkan_debug_callback.h"
+
+namespace Vulkan {
+namespace {
+VkBool32 Callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
+                  VkDebugUtilsMessageTypeFlagsEXT type,
+                  const VkDebugUtilsMessengerCallbackDataEXT* data,
+                  [[maybe_unused]] void* user_data) {
+    const std::string_view message{data->pMessage};
+    if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) {
+        LOG_CRITICAL(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT) {
+        LOG_WARNING(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT) {
+        LOG_INFO(Render_Vulkan, "{}", message);
+    } else if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT) {
+        LOG_DEBUG(Render_Vulkan, "{}", message);
+    }
+    return VK_FALSE;
+}
+} // Anonymous namespace
+
+vk::DebugUtilsMessenger CreateDebugCallback(const vk::Instance& instance) {
+    return instance.CreateDebugUtilsMessenger(VkDebugUtilsMessengerCreateInfoEXT{
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
+        .pNext = nullptr,
+        .flags = 0,
+        .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
+                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT,
+        .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
+                       VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
+                       VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
+        .pfnUserCallback = Callback,
+    });
+}
+
+} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_debug_callback.h b/src/video_core/vulkan_common/vulkan_debug_callback.h
new file mode 100644
index 000000000..2efcd244c
--- /dev/null
+++ b/src/video_core/vulkan_common/vulkan_debug_callback.h
@@ -0,0 +1,11 @@
+// Copyright 2020 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+namespace Vulkan {
+
+vk::DebugUtilsMessenger CreateDebugCallback(const vk::Instance& instance);
+
+} // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_instance.cpp b/src/video_core/vulkan_common/vulkan_instance.cpp
index c19f93e0a..d3d8630e5 100644
--- a/src/video_core/vulkan_common/vulkan_instance.cpp
+++ b/src/video_core/vulkan_common/vulkan_instance.cpp
@@ -117,21 +117,20 @@ std::pair<vk::Instance, u32> CreateInstance(Common::DynamicLibrary& library,
                                             bool enable_debug_utils, bool enable_layers) {
     if (!library.IsOpen()) {
         LOG_ERROR(Render_Vulkan, "Vulkan library not available");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     if (!library.GetSymbol("vkGetInstanceProcAddr", &dld.vkGetInstanceProcAddr)) {
         LOG_ERROR(Render_Vulkan, "vkGetInstanceProcAddr not present in Vulkan");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     if (!vk::Load(dld)) {
         LOG_ERROR(Render_Vulkan, "Failed to load Vulkan function pointers");
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     const std::vector<const char*> extensions = RequiredExtensions(window_type, enable_debug_utils);
     if (!AreExtensionsSupported(dld, extensions)) {
-        return {};
+        throw vk::Exception(VK_ERROR_EXTENSION_NOT_PRESENT);
     }
-
     std::vector<const char*> layers = Layers(enable_layers);
     RemoveUnavailableLayers(dld, layers);
 
@@ -139,12 +138,9 @@ std::pair<vk::Instance, u32> CreateInstance(Common::DynamicLibrary& library,
     const u32 version = std::min(vk::AvailableVersion(dld), VK_API_VERSION_1_1);
 
     vk::Instance instance = vk::Instance::Create(version, layers, extensions, dld);
-    if (!instance) {
-        LOG_ERROR(Render_Vulkan, "Failed to create Vulkan instance");
-        return {};
-    }
     if (!vk::Load(*instance, dld)) {
         LOG_ERROR(Render_Vulkan, "Failed to load Vulkan instance function pointers");
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
     return std::make_pair(std::move(instance), version);
 }
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp
index 478402bbd..f4177537b 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.cpp
+++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp
@@ -435,7 +435,7 @@ VkResult Free(VkDevice device, VkCommandPool handle, Span<VkCommandBuffer> buffe
 }
 
 Instance Instance::Create(u32 version, Span<const char*> layers, Span<const char*> extensions,
-                          InstanceDispatch& dispatch) noexcept {
+                          InstanceDispatch& dispatch) {
     const VkApplicationInfo application_info{
         .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
         .pNext = nullptr,
@@ -455,22 +455,17 @@ Instance Instance::Create(u32 version, Span<const char*> layers, Span<const char
         .enabledExtensionCount = extensions.size(),
         .ppEnabledExtensionNames = extensions.data(),
     };
-
     VkInstance instance;
-    if (dispatch.vkCreateInstance(&ci, nullptr, &instance) != VK_SUCCESS) {
-        // Failed to create the instance.
-        return {};
-    }
+    Check(dispatch.vkCreateInstance(&ci, nullptr, &instance));
     if (!Proc(dispatch.vkDestroyInstance, dispatch, "vkDestroyInstance", instance)) {
         // We successfully created an instance but the destroy function couldn't be loaded.
         // This is a good moment to panic.
-        return {};
+        throw vk::Exception(VK_ERROR_INITIALIZATION_FAILED);
     }
-
     return Instance(instance, dispatch);
 }
 
-std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices() {
+std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices() const {
     u32 num;
     if (dld->vkEnumeratePhysicalDevices(handle, &num, nullptr) != VK_SUCCESS) {
         return std::nullopt;
@@ -483,27 +478,11 @@ std::optional<std::vector<VkPhysicalDevice>> Instance::EnumeratePhysicalDevices(
     return std::make_optional(std::move(physical_devices));
 }
 
-DebugCallback Instance::TryCreateDebugCallback(
-    PFN_vkDebugUtilsMessengerCallbackEXT callback) noexcept {
-    const VkDebugUtilsMessengerCreateInfoEXT ci{
-        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
-        .pNext = nullptr,
-        .flags = 0,
-        .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
-                           VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT,
-        .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
-                       VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT,
-        .pfnUserCallback = callback,
-        .pUserData = nullptr,
-    };
-
-    VkDebugUtilsMessengerEXT messenger;
-    if (dld->vkCreateDebugUtilsMessengerEXT(handle, &ci, nullptr, &messenger) != VK_SUCCESS) {
-        return {};
-    }
-    return DebugCallback(messenger, handle, *dld);
+DebugUtilsMessenger Instance::CreateDebugUtilsMessenger(
+    const VkDebugUtilsMessengerCreateInfoEXT& create_info) const {
+    VkDebugUtilsMessengerEXT object;
+    Check(dld->vkCreateDebugUtilsMessengerEXT(handle, &create_info, nullptr, &object));
+    return DebugUtilsMessenger(object, handle, *dld);
 }
 
 void Buffer::BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const {
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index f9a184e00..03ca97ac0 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -555,7 +555,7 @@ private:
     const DeviceDispatch* dld = nullptr;
 };
 
-using DebugCallback = Handle<VkDebugUtilsMessengerEXT, VkInstance, InstanceDispatch>;
+using DebugUtilsMessenger = Handle<VkDebugUtilsMessengerEXT, VkInstance, InstanceDispatch>;
 using DescriptorSetLayout = Handle<VkDescriptorSetLayout, VkDevice, DeviceDispatch>;
 using DescriptorUpdateTemplateKHR = Handle<VkDescriptorUpdateTemplateKHR, VkDevice, DeviceDispatch>;
 using Pipeline = Handle<VkPipeline, VkDevice, DeviceDispatch>;
@@ -573,16 +573,19 @@ class Instance : public Handle<VkInstance, NoOwner, InstanceDispatch> {
     using Handle<VkInstance, NoOwner, InstanceDispatch>::Handle;
 
 public:
-    /// Creates a Vulkan instance. Use "operator bool" for error handling.
+    /// Creates a Vulkan instance.
+    /// @throw Exception on initialization error.
     static Instance Create(u32 version, Span<const char*> layers, Span<const char*> extensions,
-                           InstanceDispatch& dispatch) noexcept;
+                           InstanceDispatch& dispatch);
 
     /// Enumerates physical devices.
     /// @return Physical devices and an empty handle on failure.
-    std::optional<std::vector<VkPhysicalDevice>> EnumeratePhysicalDevices();
+    std::optional<std::vector<VkPhysicalDevice>> EnumeratePhysicalDevices() const;
 
-    /// Tries to create a debug callback messenger. Returns an empty handle on failure.
-    DebugCallback TryCreateDebugCallback(PFN_vkDebugUtilsMessengerCallbackEXT callback) noexcept;
+    /// Creates a debug callback messenger.
+    /// @throw Exception on creation failure.
+    DebugUtilsMessenger CreateDebugUtilsMessenger(
+        const VkDebugUtilsMessengerCreateInfoEXT& create_info) const;
 };
 
 class Queue {