vk_sampler_cache: Use VK_EXT_custom_border_color when available

This should fix grass interactions on Breath of the Wild on Vulkan.
It is currently untested against validation layers.

Nvidia's Windows 443.09 beta driver or Linux 440.66.12 is required for
now.

3 files changed, 44 insertions, 2 deletions

diff --git a/src/video_core/renderer_vulkan/vk_device.cpp b/src/video_core/renderer_vulkan/vk_device.cpp
index 0e4bbca97..09ddfa59c 100644
--- a/src/video_core/renderer_vulkan/vk_device.cpp
+++ b/src/video_core/renderer_vulkan/vk_device.cpp
@@ -293,6 +293,17 @@ bool VKDevice::Create() {
         LOG_INFO(Render_Vulkan, "Device doesn't support transform feedbacks");
     }
 
+    VkPhysicalDeviceCustomBorderColorFeaturesEXT custom_border;
+    if (ext_custom_border_color) {
+        custom_border.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT;
+        custom_border.pNext = nullptr;
+        custom_border.customBorderColors = VK_TRUE;
+        custom_border.customBorderColorWithoutFormat = VK_TRUE;
+        SetNext(next, custom_border);
+    } else {
+        LOG_INFO(Render_Vulkan, "Device doesn't support custom border colors");
+    }
+
     if (!ext_depth_range_unrestricted) {
         LOG_INFO(Render_Vulkan, "Device doesn't support depth range unrestricted");
     }
@@ -520,6 +531,7 @@ std::vector<const char*> VKDevice::LoadExtensions() {
     bool has_khr_shader_float16_int8{};
     bool has_ext_subgroup_size_control{};
     bool has_ext_transform_feedback{};
+    bool has_ext_custom_border_color{};
     for (const auto& extension : physical.EnumerateDeviceExtensionProperties()) {
         Test(extension, khr_uniform_buffer_standard_layout,
              VK_KHR_UNIFORM_BUFFER_STANDARD_LAYOUT_EXTENSION_NAME, true);
@@ -534,6 +546,8 @@ std::vector<const char*> VKDevice::LoadExtensions() {
              false);
         Test(extension, has_ext_transform_feedback, VK_EXT_TRANSFORM_FEEDBACK_EXTENSION_NAME,
              false);
+        Test(extension, has_ext_custom_border_color, VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME,
+             false);
         if (Settings::values.renderer_debug) {
             Test(extension, nv_device_diagnostics_config,
                  VK_NV_DEVICE_DIAGNOSTICS_CONFIG_EXTENSION_NAME, true);
@@ -606,6 +620,19 @@ std::vector<const char*> VKDevice::LoadExtensions() {
         }
     }
 
+    if (has_ext_custom_border_color) {
+        VkPhysicalDeviceCustomBorderColorFeaturesEXT border_features;
+        border_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT;
+        border_features.pNext = nullptr;
+        features.pNext = &border_features;
+        physical.GetFeatures2KHR(features);
+
+        if (border_features.customBorderColors && border_features.customBorderColorWithoutFormat) {
+            extensions.push_back(VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
+            ext_custom_border_color = true;
+        }
+    }
+
     return extensions;
 }
 
diff --git a/src/video_core/renderer_vulkan/vk_device.h b/src/video_core/renderer_vulkan/vk_device.h
index c8640762d..ccdf82754 100644
--- a/src/video_core/renderer_vulkan/vk_device.h
+++ b/src/video_core/renderer_vulkan/vk_device.h
@@ -172,6 +172,11 @@ public:
         return ext_transform_feedback;
     }
 
+    /// Returns true if the device supports VK_EXT_custom_border_color.
+    bool IsExtCustomBorderColorSupported() const {
+        return ext_custom_border_color;
+    }
+
     /// Returns the vendor name reported from Vulkan.
     std::string_view GetVendorName() const {
         return vendor_name;
@@ -227,6 +232,7 @@ private:
     bool ext_depth_range_unrestricted{};       ///< Support for VK_EXT_depth_range_unrestricted.
     bool ext_shader_viewport_index_layer{};    ///< Support for VK_EXT_shader_viewport_index_layer.
     bool ext_transform_feedback{};             ///< Support for VK_EXT_transform_feedback.
+    bool ext_custom_border_color{};            ///< Support for VK_EXT_custom_border_color.
     bool nv_device_diagnostics_config{};       ///< Support for VK_NV_device_diagnostics_config.
 
     // Telemetry parameters
diff --git a/src/video_core/renderer_vulkan/vk_sampler_cache.cpp b/src/video_core/renderer_vulkan/vk_sampler_cache.cpp
index 2687d8d95..e6f2fa553 100644
--- a/src/video_core/renderer_vulkan/vk_sampler_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_sampler_cache.cpp
@@ -39,9 +39,18 @@ VKSamplerCache::VKSamplerCache(const VKDevice& device) : device{device} {}
 VKSamplerCache::~VKSamplerCache() = default;
 
 vk::Sampler VKSamplerCache::CreateSampler(const Tegra::Texture::TSCEntry& tsc) const {
+    const bool arbitrary_borders = device.IsExtCustomBorderColorSupported();
+    const std::array color = tsc.GetBorderColor();
+
+    VkSamplerCustomBorderColorCreateInfoEXT border;
+    border.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
+    border.pNext = nullptr;
+    border.format = VK_FORMAT_UNDEFINED;
+    std::memcpy(&border.customBorderColor, color.data(), sizeof(color));
+
     VkSamplerCreateInfo ci;
     ci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
-    ci.pNext = nullptr;
+    ci.pNext = arbitrary_borders ? &border : nullptr;
     ci.flags = 0;
     ci.magFilter = MaxwellToVK::Sampler::Filter(tsc.mag_filter);
     ci.minFilter = MaxwellToVK::Sampler::Filter(tsc.min_filter);
@@ -56,7 +65,7 @@ vk::Sampler VKSamplerCache::CreateSampler(const Tegra::Texture::TSCEntry& tsc) c
     ci.compareOp = MaxwellToVK::Sampler::DepthCompareFunction(tsc.depth_compare_func);
     ci.minLod = tsc.GetMinLod();
     ci.maxLod = tsc.GetMaxLod();
-    ci.borderColor = ConvertBorderColor(tsc.GetBorderColor());
+    ci.borderColor = arbitrary_borders ? VK_BORDER_COLOR_INT_CUSTOM_EXT : ConvertBorderColor(color);
     ci.unnormalizedCoordinates = VK_FALSE;
     return device.GetLogical().CreateSampler(ci);
 }