Merge pull request #10473 from GPUCode/vma

Use vulkan memory allocator

6 files changed, 299 insertions, 147 deletions

diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index b11abe311..e4ca65b58 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -22,6 +22,8 @@
 #include <adrenotools/bcenabler.h>
 #endif
 
+#include <vk_mem_alloc.h>
+
 namespace Vulkan {
 using namespace Common::Literals;
 namespace {
@@ -596,9 +598,31 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
 
     graphics_queue = logical.GetQueue(graphics_family);
     present_queue = logical.GetQueue(present_family);
+
+    VmaVulkanFunctions functions{};
+    functions.vkGetInstanceProcAddr = dld.vkGetInstanceProcAddr;
+    functions.vkGetDeviceProcAddr = dld.vkGetDeviceProcAddr;
+
+    const VmaAllocatorCreateInfo allocator_info = {
+        .flags = VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT,
+        .physicalDevice = physical,
+        .device = *logical,
+        .preferredLargeHeapBlockSize = 0,
+        .pAllocationCallbacks = nullptr,
+        .pDeviceMemoryCallbacks = nullptr,
+        .pHeapSizeLimit = nullptr,
+        .pVulkanFunctions = &functions,
+        .instance = instance,
+        .vulkanApiVersion = VK_API_VERSION_1_1,
+        .pTypeExternalMemoryHandleTypes = nullptr,
+    };
+
+    vk::Check(vmaCreateAllocator(&allocator_info, &allocator));
 }
 
-Device::~Device() = default;
+Device::~Device() {
+    vmaDestroyAllocator(allocator);
+}
 
 VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage,
                                     FormatType format_type) const {
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index 0b634a876..b84af3dfb 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -14,6 +14,8 @@
 #include "common/settings.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
+VK_DEFINE_HANDLE(VmaAllocator)
+
 // Define all features which may be used by the implementation here.
 // Vulkan version in the macro describes the minimum version required for feature availability.
 // If the Vulkan version is lower than the required version, the named extension is required.
@@ -199,6 +201,11 @@ public:
         return dld;
     }
 
+    /// Returns the VMA allocator.
+    VmaAllocator GetAllocator() const {
+        return allocator;
+    }
+
     /// Returns the logical device.
     const vk::Device& GetLogical() const {
         return logical;
@@ -630,6 +637,7 @@ private:
 
 private:
     VkInstance instance;         ///< Vulkan instance.
+    VmaAllocator allocator;      ///< VMA allocator.
     vk::DeviceDispatch dld;      ///< Device function pointers.
     vk::PhysicalDevice physical; ///< Physical device.
     vk::Device logical;          ///< Logical device.
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index e28a556f8..a2ef0efa4 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -6,8 +6,6 @@
 #include <optional>
 #include <vector>
 
-#include <glad/glad.h>
-
 #include "common/alignment.h"
 #include "common/assert.h"
 #include "common/common_types.h"
@@ -17,6 +15,8 @@
 #include "video_core/vulkan_common/vulkan_memory_allocator.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
+#include <vk_mem_alloc.h>
+
 namespace Vulkan {
 namespace {
 struct Range {
@@ -49,22 +49,45 @@ struct Range {
     case MemoryUsage::Download:
         return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
                VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
+    case MemoryUsage::Stream:
+        return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+               VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
     }
     ASSERT_MSG(false, "Invalid memory usage={}", usage);
     return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
 }
 
-constexpr VkExportMemoryAllocateInfo EXPORT_ALLOCATE_INFO{
-    .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
-    .pNext = nullptr,
-#ifdef _WIN32
-    .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT,
-#elif __unix__
-    .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
-#else
-    .handleTypes = 0,
-#endif
-};
+[[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferedVmaFlags(MemoryUsage usage) {
+    return usage != MemoryUsage::DeviceLocal ? VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
+                                             : VkMemoryPropertyFlagBits{};
+}
+
+[[nodiscard]] VmaAllocationCreateFlags MemoryUsageVmaFlags(MemoryUsage usage) {
+    switch (usage) {
+    case MemoryUsage::Upload:
+    case MemoryUsage::Stream:
+        return VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
+    case MemoryUsage::Download:
+        return VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT;
+    case MemoryUsage::DeviceLocal:
+        return VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
+               VMA_ALLOCATION_CREATE_HOST_ACCESS_ALLOW_TRANSFER_INSTEAD_BIT;
+    }
+    return {};
+}
+
+[[nodiscard]] VmaMemoryUsage MemoryUsageVma(MemoryUsage usage) {
+    switch (usage) {
+    case MemoryUsage::DeviceLocal:
+    case MemoryUsage::Stream:
+        return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
+    case MemoryUsage::Upload:
+    case MemoryUsage::Download:
+        return VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
+    }
+    return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
+}
+
 } // Anonymous namespace
 
 class MemoryAllocation {
@@ -74,14 +97,6 @@ public:
         : allocator{allocator_}, memory{std::move(memory_)}, allocation_size{allocation_size_},
           property_flags{properties}, shifted_memory_type{1U << type} {}
 
-#if defined(_WIN32) || defined(__unix__)
-    ~MemoryAllocation() {
-        if (owning_opengl_handle != 0) {
-            glDeleteMemoryObjectsEXT(1, &owning_opengl_handle);
-        }
-    }
-#endif
-
     MemoryAllocation& operator=(const MemoryAllocation&) = delete;
     MemoryAllocation(const MemoryAllocation&) = delete;
 
@@ -120,31 +135,6 @@ public:
         return memory_mapped_span;
     }
 
-#ifdef _WIN32
-    [[nodiscard]] u32 ExportOpenGLHandle() {
-        if (!owning_opengl_handle) {
-            glCreateMemoryObjectsEXT(1, &owning_opengl_handle);
-            glImportMemoryWin32HandleEXT(owning_opengl_handle, allocation_size,
-                                         GL_HANDLE_TYPE_OPAQUE_WIN32_EXT,
-                                         memory.GetMemoryWin32HandleKHR());
-        }
-        return owning_opengl_handle;
-    }
-#elif __unix__
-    [[nodiscard]] u32 ExportOpenGLHandle() {
-        if (!owning_opengl_handle) {
-            glCreateMemoryObjectsEXT(1, &owning_opengl_handle);
-            glImportMemoryFdEXT(owning_opengl_handle, allocation_size, GL_HANDLE_TYPE_OPAQUE_FD_EXT,
-                                memory.GetMemoryFdKHR());
-        }
-        return owning_opengl_handle;
-    }
-#else
-    [[nodiscard]] u32 ExportOpenGLHandle() {
-        return 0;
-    }
-#endif
-
     /// Returns whether this allocation is compatible with the arguments.
     [[nodiscard]] bool IsCompatible(VkMemoryPropertyFlags flags, u32 type_mask) const {
         return (flags & property_flags) == flags && (type_mask & shifted_memory_type) != 0;
@@ -182,9 +172,6 @@ private:
     const u32 shifted_memory_type;              ///< Shifted Vulkan memory type.
     std::vector<Range> commits;                 ///< All commit ranges done from this allocation.
     std::span<u8> memory_mapped_span; ///< Memory mapped span. Empty if not queried before.
-#if defined(_WIN32) || defined(__unix__)
-    u32 owning_opengl_handle{}; ///< Owning OpenGL memory object handle.
-#endif
 };
 
 MemoryCommit::MemoryCommit(MemoryAllocation* allocation_, VkDeviceMemory memory_, u64 begin_,
@@ -216,24 +203,70 @@ std::span<u8> MemoryCommit::Map() {
     return span;
 }
 
-u32 MemoryCommit::ExportOpenGLHandle() const {
-    return allocation->ExportOpenGLHandle();
-}
-
 void MemoryCommit::Release() {
     if (allocation) {
         allocation->Free(begin);
     }
 }
 
-MemoryAllocator::MemoryAllocator(const Device& device_, bool export_allocations_)
-    : device{device_}, properties{device_.GetPhysical().GetMemoryProperties().memoryProperties},
-      export_allocations{export_allocations_},
+MemoryAllocator::MemoryAllocator(const Device& device_)
+    : device{device_}, allocator{device.GetAllocator()},
+      properties{device_.GetPhysical().GetMemoryProperties().memoryProperties},
       buffer_image_granularity{
           device_.GetPhysical().GetProperties().limits.bufferImageGranularity} {}
 
 MemoryAllocator::~MemoryAllocator() = default;
 
+vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo& ci) const {
+    const VmaAllocationCreateInfo alloc_ci = {
+        .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT,
+        .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
+        .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
+        .preferredFlags = 0,
+        .memoryTypeBits = 0,
+        .pool = VK_NULL_HANDLE,
+        .pUserData = nullptr,
+        .priority = 0.f,
+    };
+
+    VkImage handle{};
+    VmaAllocation allocation{};
+
+    vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, nullptr));
+
+    return vk::Image(handle, *device.GetLogical(), allocator, allocation,
+                     device.GetDispatchLoader());
+}
+
+vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo& ci, MemoryUsage usage) const {
+    const VmaAllocationCreateInfo alloc_ci = {
+        .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT |
+                 MemoryUsageVmaFlags(usage),
+        .usage = MemoryUsageVma(usage),
+        .requiredFlags = 0,
+        .preferredFlags = MemoryUsagePreferedVmaFlags(usage),
+        .memoryTypeBits = 0,
+        .pool = VK_NULL_HANDLE,
+        .pUserData = nullptr,
+        .priority = 0.f,
+    };
+
+    VkBuffer handle{};
+    VmaAllocationInfo alloc_info{};
+    VmaAllocation allocation{};
+    VkMemoryPropertyFlags property_flags{};
+
+    vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
+    vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags);
+
+    u8* data = reinterpret_cast<u8*>(alloc_info.pMappedData);
+    const std::span<u8> mapped_data = data ? std::span<u8>{data, ci.size} : std::span<u8>{};
+    const bool is_coherent = property_flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+
+    return vk::Buffer(handle, *device.GetLogical(), allocator, allocation, mapped_data, is_coherent,
+                      device.GetDispatchLoader());
+}
+
 MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements& requirements, MemoryUsage usage) {
     // Find the fastest memory flags we can afford with the current requirements
     const u32 type_mask = requirements.memoryTypeBits;
@@ -253,25 +286,11 @@ MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements& requirements, M
     return TryCommit(requirements, flags).value();
 }
 
-MemoryCommit MemoryAllocator::Commit(const vk::Buffer& buffer, MemoryUsage usage) {
-    auto commit = Commit(device.GetLogical().GetBufferMemoryRequirements(*buffer), usage);
-    buffer.BindMemory(commit.Memory(), commit.Offset());
-    return commit;
-}
-
-MemoryCommit MemoryAllocator::Commit(const vk::Image& image, MemoryUsage usage) {
-    VkMemoryRequirements requirements = device.GetLogical().GetImageMemoryRequirements(*image);
-    requirements.size = Common::AlignUp(requirements.size, buffer_image_granularity);
-    auto commit = Commit(requirements, usage);
-    image.BindMemory(commit.Memory(), commit.Offset());
-    return commit;
-}
-
 bool MemoryAllocator::TryAllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size) {
     const u32 type = FindType(flags, type_mask).value();
     vk::DeviceMemory memory = device.GetLogical().TryAllocateMemory({
         .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
-        .pNext = export_allocations ? &EXPORT_ALLOCATE_INFO : nullptr,
+        .pNext = nullptr,
         .allocationSize = size,
         .memoryTypeIndex = type,
     });
@@ -342,16 +361,4 @@ std::optional<u32> MemoryAllocator::FindType(VkMemoryPropertyFlags flags, u32 ty
     return std::nullopt;
 }
 
-bool IsHostVisible(MemoryUsage usage) noexcept {
-    switch (usage) {
-    case MemoryUsage::DeviceLocal:
-        return false;
-    case MemoryUsage::Upload:
-    case MemoryUsage::Download:
-        return true;
-    }
-    ASSERT_MSG(false, "Invalid memory usage={}", usage);
-    return false;
-}
-
 } // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index a5bff03fe..f449bc8d0 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -9,6 +9,8 @@
 #include "common/common_types.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
+VK_DEFINE_HANDLE(VmaAllocator)
+
 namespace Vulkan {
 
 class Device;
@@ -17,9 +19,11 @@ class MemoryAllocation;
 
 /// Hints and requirements for the backing memory type of a commit
 enum class MemoryUsage {
-    DeviceLocal, ///< Hints device local usages, fastest memory type to read and write from the GPU
+    DeviceLocal, ///< Requests device local host visible buffer, falling back to device local
+                 ///< memory.
     Upload,      ///< Requires a host visible memory type optimized for CPU to GPU uploads
     Download,    ///< Requires a host visible memory type optimized for GPU to CPU readbacks
+    Stream,      ///< Requests device local host visible buffer, falling back host memory.
 };
 
 /// Ownership handle of a memory commitment.
@@ -41,9 +45,6 @@ public:
     /// It will map the backing allocation if it hasn't been mapped before.
     std::span<u8> Map();
 
-    /// Returns an non-owning OpenGL handle, creating one if it doesn't exist.
-    u32 ExportOpenGLHandle() const;
-
     /// Returns the Vulkan memory handler.
     VkDeviceMemory Memory() const {
         return memory;
@@ -74,16 +75,19 @@ public:
      * Construct memory allocator
      *
      * @param device_             Device to allocate from
-     * @param export_allocations_ True when allocations have to be exported
      *
      * @throw vk::Exception on failure
      */
-    explicit MemoryAllocator(const Device& device_, bool export_allocations_);
+    explicit MemoryAllocator(const Device& device_);
     ~MemoryAllocator();
 
     MemoryAllocator& operator=(const MemoryAllocator&) = delete;
     MemoryAllocator(const MemoryAllocator&) = delete;
 
+    vk::Image CreateImage(const VkImageCreateInfo& ci) const;
+
+    vk::Buffer CreateBuffer(const VkBufferCreateInfo& ci, MemoryUsage usage) const;
+
     /**
      * Commits a memory with the specified requirements.
      *
@@ -97,9 +101,6 @@ public:
     /// Commits memory required by the buffer and binds it.
     MemoryCommit Commit(const vk::Buffer& buffer, MemoryUsage usage);
 
-    /// Commits memory required by the image and binds it.
-    MemoryCommit Commit(const vk::Image& image, MemoryUsage usage);
-
 private:
     /// Tries to allocate a chunk of memory.
     bool TryAllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size);
@@ -117,15 +118,12 @@ private:
     /// Returns index to the fastest memory type compatible with the passed requirements.
     std::optional<u32> FindType(VkMemoryPropertyFlags flags, u32 type_mask) const;
 
-    const Device& device;                              ///< Device handle.
-    const VkPhysicalDeviceMemoryProperties properties; ///< Physical device properties.
-    const bool export_allocations; ///< True when memory allocations have to be exported.
+    const Device& device;                                       ///< Device handle.
+    VmaAllocator allocator;                                     ///< Vma allocator.
+    const VkPhysicalDeviceMemoryProperties properties;          ///< Physical device properties.
     std::vector<std::unique_ptr<MemoryAllocation>> allocations; ///< Current allocations.
     VkDeviceSize buffer_image_granularity; // The granularity for adjacent offsets between buffers
                                            // and optimal images
 };
 
-/// Returns true when a memory usage is guaranteed to be host visible.
-bool IsHostVisible(MemoryUsage usage) noexcept;
-
 } // namespace Vulkan
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp
index 336f53700..28fcb21a0 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.cpp
+++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp
@@ -12,6 +12,8 @@
 
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
+#include <vk_mem_alloc.h>
+
 namespace Vulkan::vk {
 
 namespace {
@@ -547,24 +549,40 @@ DebugUtilsMessenger Instance::CreateDebugUtilsMessenger(
     return DebugUtilsMessenger(object, handle, *dld);
 }
 
-void Buffer::BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const {
-    Check(dld->vkBindBufferMemory(owner, handle, memory, offset));
+void Image::SetObjectNameEXT(const char* name) const {
+    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name);
 }
 
-void Buffer::SetObjectNameEXT(const char* name) const {
-    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER, name);
+void Image::Release() const noexcept {
+    if (handle) {
+        vmaDestroyImage(allocator, handle, allocation);
+    }
 }
 
-void BufferView::SetObjectNameEXT(const char* name) const {
-    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER_VIEW, name);
+void Buffer::Flush() const {
+    if (!is_coherent) {
+        vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE);
+    }
 }
 
-void Image::BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const {
-    Check(dld->vkBindImageMemory(owner, handle, memory, offset));
+void Buffer::Invalidate() const {
+    if (!is_coherent) {
+        vmaInvalidateAllocation(allocator, allocation, 0, VK_WHOLE_SIZE);
+    }
 }
 
-void Image::SetObjectNameEXT(const char* name) const {
-    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name);
+void Buffer::SetObjectNameEXT(const char* name) const {
+    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER, name);
+}
+
+void Buffer::Release() const noexcept {
+    if (handle) {
+        vmaDestroyBuffer(allocator, handle, allocation);
+    }
+}
+
+void BufferView::SetObjectNameEXT(const char* name) const {
+    SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_BUFFER_VIEW, name);
 }
 
 void ImageView::SetObjectNameEXT(const char* name) const {
@@ -701,24 +719,12 @@ Queue Device::GetQueue(u32 family_index) const noexcept {
     return Queue(queue, *dld);
 }
 
-Buffer Device::CreateBuffer(const VkBufferCreateInfo& ci) const {
-    VkBuffer object;
-    Check(dld->vkCreateBuffer(handle, &ci, nullptr, &object));
-    return Buffer(object, handle, *dld);
-}
-
 BufferView Device::CreateBufferView(const VkBufferViewCreateInfo& ci) const {
     VkBufferView object;
     Check(dld->vkCreateBufferView(handle, &ci, nullptr, &object));
     return BufferView(object, handle, *dld);
 }
 
-Image Device::CreateImage(const VkImageCreateInfo& ci) const {
-    VkImage object;
-    Check(dld->vkCreateImage(handle, &ci, nullptr, &object));
-    return Image(object, handle, *dld);
-}
-
 ImageView Device::CreateImageView(const VkImageViewCreateInfo& ci) const {
     VkImageView object;
     Check(dld->vkCreateImageView(handle, &ci, nullptr, &object));
diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h
index 4ff328a21..44fce47a5 100644
--- a/src/video_core/vulkan_common/vulkan_wrapper.h
+++ b/src/video_core/vulkan_common/vulkan_wrapper.h
@@ -32,6 +32,9 @@
 #pragma warning(disable : 26812) // Disable prefer enum class over enum
 #endif
 
+VK_DEFINE_HANDLE(VmaAllocator)
+VK_DEFINE_HANDLE(VmaAllocation)
+
 namespace Vulkan::vk {
 
 /**
@@ -616,6 +619,138 @@ public:
     }
 };
 
+class Image {
+public:
+    explicit Image(VkImage handle_, VkDevice owner_, VmaAllocator allocator_,
+                   VmaAllocation allocation_, const DeviceDispatch& dld_) noexcept
+        : handle{handle_}, owner{owner_}, allocator{allocator_},
+          allocation{allocation_}, dld{&dld_} {}
+    Image() = default;
+
+    Image(const Image&) = delete;
+    Image& operator=(const Image&) = delete;
+
+    Image(Image&& rhs) noexcept
+        : handle{std::exchange(rhs.handle, nullptr)}, owner{rhs.owner}, allocator{rhs.allocator},
+          allocation{rhs.allocation}, dld{rhs.dld} {}
+
+    Image& operator=(Image&& rhs) noexcept {
+        Release();
+        handle = std::exchange(rhs.handle, nullptr);
+        owner = rhs.owner;
+        allocator = rhs.allocator;
+        allocation = rhs.allocation;
+        dld = rhs.dld;
+        return *this;
+    }
+
+    ~Image() noexcept {
+        Release();
+    }
+
+    VkImage operator*() const noexcept {
+        return handle;
+    }
+
+    void reset() noexcept {
+        Release();
+        handle = nullptr;
+    }
+
+    explicit operator bool() const noexcept {
+        return handle != nullptr;
+    }
+
+    void SetObjectNameEXT(const char* name) const;
+
+private:
+    void Release() const noexcept;
+
+    VkImage handle = nullptr;
+    VkDevice owner = nullptr;
+    VmaAllocator allocator = nullptr;
+    VmaAllocation allocation = nullptr;
+    const DeviceDispatch* dld = nullptr;
+};
+
+class Buffer {
+public:
+    explicit Buffer(VkBuffer handle_, VkDevice owner_, VmaAllocator allocator_,
+                    VmaAllocation allocation_, std::span<u8> mapped_, bool is_coherent_,
+                    const DeviceDispatch& dld_) noexcept
+        : handle{handle_}, owner{owner_}, allocator{allocator_},
+          allocation{allocation_}, mapped{mapped_}, is_coherent{is_coherent_}, dld{&dld_} {}
+    Buffer() = default;
+
+    Buffer(const Buffer&) = delete;
+    Buffer& operator=(const Buffer&) = delete;
+
+    Buffer(Buffer&& rhs) noexcept
+        : handle{std::exchange(rhs.handle, nullptr)}, owner{rhs.owner}, allocator{rhs.allocator},
+          allocation{rhs.allocation}, mapped{rhs.mapped},
+          is_coherent{rhs.is_coherent}, dld{rhs.dld} {}
+
+    Buffer& operator=(Buffer&& rhs) noexcept {
+        Release();
+        handle = std::exchange(rhs.handle, nullptr);
+        owner = rhs.owner;
+        allocator = rhs.allocator;
+        allocation = rhs.allocation;
+        mapped = rhs.mapped;
+        is_coherent = rhs.is_coherent;
+        dld = rhs.dld;
+        return *this;
+    }
+
+    ~Buffer() noexcept {
+        Release();
+    }
+
+    VkBuffer operator*() const noexcept {
+        return handle;
+    }
+
+    void reset() noexcept {
+        Release();
+        handle = nullptr;
+    }
+
+    explicit operator bool() const noexcept {
+        return handle != nullptr;
+    }
+
+    /// Returns the host mapped memory, an empty span otherwise.
+    std::span<u8> Mapped() noexcept {
+        return mapped;
+    }
+
+    std::span<const u8> Mapped() const noexcept {
+        return mapped;
+    }
+
+    /// Returns true if the buffer is mapped to the host.
+    bool IsHostVisible() const noexcept {
+        return !mapped.empty();
+    }
+
+    void Flush() const;
+
+    void Invalidate() const;
+
+    void SetObjectNameEXT(const char* name) const;
+
+private:
+    void Release() const noexcept;
+
+    VkBuffer handle = nullptr;
+    VkDevice owner = nullptr;
+    VmaAllocator allocator = nullptr;
+    VmaAllocation allocation = nullptr;
+    std::span<u8> mapped = {};
+    bool is_coherent = false;
+    const DeviceDispatch* dld = nullptr;
+};
+
 class Queue {
 public:
     /// Construct an empty queue handle.
@@ -639,17 +774,6 @@ private:
     const DeviceDispatch* dld = nullptr;
 };
 
-class Buffer : public Handle<VkBuffer, VkDevice, DeviceDispatch> {
-    using Handle<VkBuffer, VkDevice, DeviceDispatch>::Handle;
-
-public:
-    /// Attaches a memory allocation.
-    void BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const;
-
-    /// Set object name.
-    void SetObjectNameEXT(const char* name) const;
-};
-
 class BufferView : public Handle<VkBufferView, VkDevice, DeviceDispatch> {
     using Handle<VkBufferView, VkDevice, DeviceDispatch>::Handle;
 
@@ -658,17 +782,6 @@ public:
     void SetObjectNameEXT(const char* name) const;
 };
 
-class Image : public Handle<VkImage, VkDevice, DeviceDispatch> {
-    using Handle<VkImage, VkDevice, DeviceDispatch>::Handle;
-
-public:
-    /// Attaches a memory allocation.
-    void BindMemory(VkDeviceMemory memory, VkDeviceSize offset) const;
-
-    /// Set object name.
-    void SetObjectNameEXT(const char* name) const;
-};
-
 class ImageView : public Handle<VkImageView, VkDevice, DeviceDispatch> {
     using Handle<VkImageView, VkDevice, DeviceDispatch>::Handle;
 
@@ -840,12 +953,8 @@ public:
 
     Queue GetQueue(u32 family_index) const noexcept;
 
-    Buffer CreateBuffer(const VkBufferCreateInfo& ci) const;
-
     BufferView CreateBufferView(const VkBufferViewCreateInfo& ci) const;
 
-    Image CreateImage(const VkImageCreateInfo& ci) const;
-
     ImageView CreateImageView(const VkImageViewCreateInfo& ci) const;
 
     Semaphore CreateSemaphore() const;