vulkan_memory_allocator: Allow textures to be allocated in host memory

Allow Vulkan's allocator to use host memory when there's no more device
local memory. This delays OOM, but it will eventually still happen.

2 files changed, 43 insertions, 31 deletions

diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index fa37aa79a..5edd06ebc 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -53,6 +53,18 @@ struct Range {
     UNREACHABLE_MSG("Invalid memory usage={}", usage);
     return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
 }
+
+constexpr VkExportMemoryAllocateInfo EXPORT_ALLOCATE_INFO{
+    .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
+    .pNext = nullptr,
+#ifdef _WIN32
+    .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT,
+#elif __unix__
+    .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
+#else
+    .handleTypes = 0,
+#endif
+};
 } // Anonymous namespace
 
 class MemoryAllocation {
@@ -131,7 +143,7 @@ public:
 
     /// Returns whether this allocation is compatible with the arguments.
     [[nodiscard]] bool IsCompatible(VkMemoryPropertyFlags flags, u32 type_mask) const {
-        return (flags & property_flags) && (type_mask & shifted_memory_type) != 0;
+        return (flags & property_flags) == property_flags && (type_mask & shifted_memory_type) != 0;
     }
 
 private:
@@ -217,14 +229,18 @@ MemoryAllocator::~MemoryAllocator() = default;
 
 MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements& requirements, MemoryUsage usage) {
     // Find the fastest memory flags we can afford with the current requirements
-    const VkMemoryPropertyFlags flags = MemoryPropertyFlags(requirements.memoryTypeBits, usage);
+    const u32 type_mask = requirements.memoryTypeBits;
+    const VkMemoryPropertyFlags usage_flags = MemoryUsagePropertyFlags(usage);
+    const VkMemoryPropertyFlags flags = MemoryPropertyFlags(type_mask, usage_flags);
     if (std::optional<MemoryCommit> commit = TryCommit(requirements, flags)) {
         return std::move(*commit);
     }
     // Commit has failed, allocate more memory.
-    // TODO(Rodrigo): Handle out of memory situations in some way like flushing to guest memory.
-    AllocMemory(flags, requirements.memoryTypeBits, AllocationChunkSize(requirements.size));
-
+    const u64 chunk_size = AllocationChunkSize(requirements.size);
+    if (!TryAllocMemory(flags, type_mask, chunk_size)) {
+        // TODO(Rodrigo): Handle out of memory situations in some way like flushing to guest memory.
+        throw vk::Exception(VK_ERROR_OUT_OF_DEVICE_MEMORY);
+    }
     // Commit again, this time it won't fail since there's a fresh allocation above.
     // If it does, there's a bug.
     return TryCommit(requirements, flags).value();
@@ -242,26 +258,25 @@ MemoryCommit MemoryAllocator::Commit(const vk::Image& image, MemoryUsage usage)
     return commit;
 }
 
-void MemoryAllocator::AllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size) {
+bool MemoryAllocator::TryAllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size) {
     const u32 type = FindType(flags, type_mask).value();
-    const VkExportMemoryAllocateInfo export_allocate_info{
-        .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
-        .pNext = nullptr,
-#ifdef _WIN32
-        .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT,
-#elif __unix__
-        .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
-#else
-        .handleTypes = 0,
-#endif
-    };
-    vk::DeviceMemory memory = device.GetLogical().AllocateMemory({
+    vk::DeviceMemory memory = device.GetLogical().TryAllocateMemory({
         .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
-        .pNext = export_allocations ? &export_allocate_info : nullptr,
+        .pNext = export_allocations ? &EXPORT_ALLOCATE_INFO : nullptr,
         .allocationSize = size,
         .memoryTypeIndex = type,
     });
+    if (!memory) {
+        if ((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
+            // Try to allocate non device local memory
+            return TryAllocMemory(flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, type_mask, size);
+        } else {
+            // RIP
+            return false;
+        }
+    }
     allocations.push_back(std::make_unique<MemoryAllocation>(std::move(memory), flags, size, type));
+    return true;
 }
 
 std::optional<MemoryCommit> MemoryAllocator::TryCommit(const VkMemoryRequirements& requirements,
@@ -274,24 +289,24 @@ std::optional<MemoryCommit> MemoryAllocator::TryCommit(const VkMemoryRequirement
             return commit;
         }
     }
+    if ((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
+        // Look for non device local commits on failure
+        return TryCommit(requirements, flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+    }
     return std::nullopt;
 }
 
-VkMemoryPropertyFlags MemoryAllocator::MemoryPropertyFlags(u32 type_mask, MemoryUsage usage) const {
-    return MemoryPropertyFlags(type_mask, MemoryUsagePropertyFlags(usage));
-}
-
 VkMemoryPropertyFlags MemoryAllocator::MemoryPropertyFlags(u32 type_mask,
                                                            VkMemoryPropertyFlags flags) const {
     if (FindType(flags, type_mask)) {
         // Found a memory type with those requirements
         return flags;
     }
-    if (flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
+    if ((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0) {
         // Remove host cached bit in case it's not supported
         return MemoryPropertyFlags(type_mask, flags & ~VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
     }
-    if (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
+    if ((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
         // Remove device local, if it's not supported by the requested resource
         return MemoryPropertyFlags(type_mask, flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
     }
@@ -302,7 +317,7 @@ VkMemoryPropertyFlags MemoryAllocator::MemoryPropertyFlags(u32 type_mask,
 std::optional<u32> MemoryAllocator::FindType(VkMemoryPropertyFlags flags, u32 type_mask) const {
     for (u32 type_index = 0; type_index < properties.memoryTypeCount; ++type_index) {
         const VkMemoryPropertyFlags type_flags = properties.memoryTypes[type_index].propertyFlags;
-        if ((type_mask & (1U << type_index)) && (type_flags & flags)) {
+        if ((type_mask & (1U << type_index)) != 0 && (type_flags & flags) == flags) {
             // The type matches in type and in the wanted properties.
             return type_index;
         }
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index d1ce29450..db12d02f4 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -101,16 +101,13 @@ public:
     MemoryCommit Commit(const vk::Image& image, MemoryUsage usage);
 
 private:
-    /// Allocates a chunk of memory.
-    void AllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size);
+    /// Tries to allocate a chunk of memory.
+    bool TryAllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size);
 
     /// Tries to allocate a memory commit.
     std::optional<MemoryCommit> TryCommit(const VkMemoryRequirements& requirements,
                                           VkMemoryPropertyFlags flags);
 
-    /// Returns the fastest compatible memory property flags from a wanted usage.
-    VkMemoryPropertyFlags MemoryPropertyFlags(u32 type_mask, MemoryUsage usage) const;
-
     /// Returns the fastest compatible memory property flags from the wanted flags.
     VkMemoryPropertyFlags MemoryPropertyFlags(u32 type_mask, VkMemoryPropertyFlags flags) const;