renderer_vulkan: Add setting to log pipeline statistics

Use VK_KHR_pipeline_executable_properties when enabled and available to
log statistics about the pipeline cache in a game.

For example, this is on Turing GPUs when generating a pipeline cache
from Super Smash Bros. Ultimate:

Average pipeline statistics
==========================================
Code size:       6433.167
Register count:    32.939

More advanced results could be presented, at the moment it's just an
average of all 3D and compute pipelines.

8 files changed, 195 insertions, 19 deletions

diff --git a/src/video_core/renderer_vulkan/pipeline_statistics.cpp b/src/video_core/renderer_vulkan/pipeline_statistics.cpp
new file mode 100644
index 000000000..bfec931a6
--- /dev/null
+++ b/src/video_core/renderer_vulkan/pipeline_statistics.cpp
@@ -0,0 +1,100 @@
+// Copyright 2021 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <string_view>
+
+#include <fmt/format.h>
+
+#include "common/common_types.h"
+#include "common/logging/log.h"
+#include "video_core/renderer_vulkan/pipeline_statistics.h"
+#include "video_core/vulkan_common/vulkan_device.h"
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+namespace Vulkan {
+
+using namespace std::string_view_literals;
+
+static u64 GetUint64(const VkPipelineExecutableStatisticKHR& statistic) {
+    switch (statistic.format) {
+    case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_INT64_KHR:
+        return static_cast<u64>(statistic.value.i64);
+    case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR:
+        return statistic.value.u64;
+    case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_FLOAT64_KHR:
+        return static_cast<u64>(statistic.value.f64);
+    default:
+        return 0;
+    }
+}
+
+PipelineStatistics::PipelineStatistics(const Device& device_) : device{device_} {}
+
+void PipelineStatistics::Collect(VkPipeline pipeline) {
+    const auto& dev{device.GetLogical()};
+    const std::vector properties{dev.GetPipelineExecutablePropertiesKHR(pipeline)};
+    const u32 num_executables{static_cast<u32>(properties.size())};
+    for (u32 executable = 0; executable < num_executables; ++executable) {
+        const auto statistics{dev.GetPipelineExecutableStatisticsKHR(pipeline, executable)};
+        if (statistics.empty()) {
+            continue;
+        }
+        Stats stage_stats;
+        for (const auto& statistic : statistics) {
+            const char* const name{statistic.name};
+            if (name == "Binary Size"sv || name == "Code size"sv || name == "Instruction Count"sv) {
+                stage_stats.code_size = GetUint64(statistic);
+            } else if (name == "Register Count"sv) {
+                stage_stats.register_count = GetUint64(statistic);
+            } else if (name == "SGPRs"sv || name == "numUsedSgprs"sv) {
+                stage_stats.sgpr_count = GetUint64(statistic);
+            } else if (name == "VGPRs"sv || name == "numUsedVgprs"sv) {
+                stage_stats.vgpr_count = GetUint64(statistic);
+            } else if (name == "Branches"sv) {
+                stage_stats.branches_count = GetUint64(statistic);
+            } else if (name == "Basic Block Count"sv) {
+                stage_stats.basic_block_count = GetUint64(statistic);
+            }
+        }
+        std::lock_guard lock{mutex};
+        collected_stats.push_back(stage_stats);
+    }
+}
+
+void PipelineStatistics::Report() const {
+    double num{};
+    Stats total;
+    {
+        std::lock_guard lock{mutex};
+        for (const Stats& stats : collected_stats) {
+            total.code_size += stats.code_size;
+            total.register_count += stats.register_count;
+            total.sgpr_count += stats.sgpr_count;
+            total.vgpr_count += stats.vgpr_count;
+            total.branches_count += stats.branches_count;
+            total.basic_block_count += stats.basic_block_count;
+        }
+        num = static_cast<double>(collected_stats.size());
+    }
+    std::string report;
+    const auto add = [&](const char* fmt, u64 value) {
+        if (value > 0) {
+            report += fmt::format(fmt::runtime(fmt), static_cast<double>(value) / num);
+        }
+    };
+    add("Code size:      {:9.03f}\n", total.code_size);
+    add("Register count: {:9.03f}\n", total.register_count);
+    add("SGPRs:          {:9.03f}\n", total.sgpr_count);
+    add("VGPRs:          {:9.03f}\n", total.vgpr_count);
+    add("Branches count: {:9.03f}\n", total.branches_count);
+    add("Basic blocks:   {:9.03f}\n", total.basic_block_count);
+
+    LOG_INFO(Render_Vulkan,
+             "\nAverage pipeline statistics\n"
+             "==========================================\n"
+             "{}\n",
+             report);
+}
+
+} // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/pipeline_statistics.h b/src/video_core/renderer_vulkan/pipeline_statistics.h
new file mode 100644
index 000000000..b61840107
--- /dev/null
+++ b/src/video_core/renderer_vulkan/pipeline_statistics.h
@@ -0,0 +1,40 @@
+// Copyright 2021 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <mutex>
+#include <vector>
+
+#include "common/common_types.h"
+#include "video_core/vulkan_common/vulkan_wrapper.h"
+
+namespace Vulkan {
+
+class Device;
+
+class PipelineStatistics {
+public:
+    explicit PipelineStatistics(const Device& device_);
+
+    void Collect(VkPipeline pipeline);
+
+    void Report() const;
+
+private:
+    struct Stats {
+        u64 code_size{};
+        u64 register_count{};
+        u64 sgpr_count{};
+        u64 vgpr_count{};
+        u64 branches_count{};
+        u64 basic_block_count{};
+    };
+
+    const Device& device;
+    mutable std::mutex mutex;
+    std::vector<Stats> collected_stats;
+};
+
+} // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
index 70b84c7a6..44faf626a 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
@@ -8,6 +8,7 @@
 #include <boost/container/small_vector.hpp>
 
 #include "video_core/renderer_vulkan/pipeline_helper.h"
+#include "video_core/renderer_vulkan/pipeline_statistics.h"
 #include "video_core/renderer_vulkan/vk_buffer_cache.h"
 #include "video_core/renderer_vulkan/vk_compute_pipeline.h"
 #include "video_core/renderer_vulkan/vk_descriptor_pool.h"
@@ -26,6 +27,7 @@ using Tegra::Texture::TexturePair;
 ComputePipeline::ComputePipeline(const Device& device_, DescriptorPool& descriptor_pool,
                                  VKUpdateDescriptorQueue& update_descriptor_queue_,
                                  Common::ThreadWorker* thread_worker,
+                                 PipelineStatistics* pipeline_statistics,
                                  VideoCore::ShaderNotify* shader_notify, const Shader::Info& info_,
                                  vk::ShaderModule spv_module_)
     : device{device_}, update_descriptor_queue{update_descriptor_queue_}, info{info_},
@@ -36,7 +38,7 @@ ComputePipeline::ComputePipeline(const Device& device_, DescriptorPool& descript
     std::copy_n(info.constant_buffer_used_sizes.begin(), uniform_buffer_sizes.size(),
                 uniform_buffer_sizes.begin());
 
-    auto func{[this, &descriptor_pool, shader_notify] {
+    auto func{[this, &descriptor_pool, shader_notify, pipeline_statistics] {
         DescriptorLayoutBuilder builder{device};
         builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT);
 
@@ -50,10 +52,14 @@ ComputePipeline::ComputePipeline(const Device& device_, DescriptorPool& descript
             .pNext = nullptr,
             .requiredSubgroupSize = GuestWarpSize,
         };
+        VkPipelineCreateFlags flags{};
+        if (device.IsKhrPipelineEexecutablePropertiesEnabled()) {
+            flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR;
+        }
         pipeline = device.GetLogical().CreateComputePipeline({
             .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
             .pNext = nullptr,
-            .flags = 0,
+            .flags = flags,
             .stage{
                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
                 .pNext = device.IsExtSubgroupSizeControlSupported() ? &subgroup_size_ci : nullptr,
@@ -67,6 +73,9 @@ ComputePipeline::ComputePipeline(const Device& device_, DescriptorPool& descript
             .basePipelineHandle = 0,
             .basePipelineIndex = 0,
         });
+        if (pipeline_statistics) {
+            pipeline_statistics->Collect(*pipeline);
+        }
         std::lock_guard lock{build_mutex};
         is_built = true;
         build_condvar.notify_one();
diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.h b/src/video_core/renderer_vulkan/vk_compute_pipeline.h
index 52fec04d3..8c4b0a301 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pipeline.h
+++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.h
@@ -25,6 +25,7 @@ class ShaderNotify;
 namespace Vulkan {
 
 class Device;
+class PipelineStatistics;
 class VKScheduler;
 
 class ComputePipeline {
@@ -32,6 +33,7 @@ public:
     explicit ComputePipeline(const Device& device, DescriptorPool& descriptor_pool,
                              VKUpdateDescriptorQueue& update_descriptor_queue,
                              Common::ThreadWorker* thread_worker,
+                             PipelineStatistics* pipeline_statistics,
                              VideoCore::ShaderNotify* shader_notify, const Shader::Info& info,
                              vk::ShaderModule spv_module);
 
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
index 18482e1d0..7c0f91007 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
@@ -11,6 +11,7 @@
 #include "common/bit_field.h"
 #include "video_core/renderer_vulkan/maxwell_to_vk.h"
 #include "video_core/renderer_vulkan/pipeline_helper.h"
+#include "video_core/renderer_vulkan/pipeline_statistics.h"
 #include "video_core/renderer_vulkan/vk_buffer_cache.h"
 #include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
 #include "video_core/renderer_vulkan/vk_render_pass_cache.h"
@@ -217,8 +218,8 @@ GraphicsPipeline::GraphicsPipeline(
     VKScheduler& scheduler_, BufferCache& buffer_cache_, TextureCache& texture_cache_,
     VideoCore::ShaderNotify* shader_notify, const Device& device_, DescriptorPool& descriptor_pool,
     VKUpdateDescriptorQueue& update_descriptor_queue_, Common::ThreadWorker* worker_thread,
-    RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key_,
-    std::array<vk::ShaderModule, NUM_STAGES> stages,
+    PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache,
+    const GraphicsPipelineCacheKey& key_, std::array<vk::ShaderModule, NUM_STAGES> stages,
     const std::array<const Shader::Info*, NUM_STAGES>& infos)
     : key{key_}, maxwell3d{maxwell3d_}, gpu_memory{gpu_memory_}, device{device_},
       texture_cache{texture_cache_}, buffer_cache{buffer_cache_}, scheduler{scheduler_},
@@ -235,7 +236,7 @@ GraphicsPipeline::GraphicsPipeline(
         enabled_uniform_buffer_masks[stage] = info->constant_buffer_mask;
         std::ranges::copy(info->constant_buffer_used_sizes, uniform_buffer_sizes[stage].begin());
     }
-    auto func{[this, shader_notify, &render_pass_cache, &descriptor_pool] {
+    auto func{[this, shader_notify, &render_pass_cache, &descriptor_pool, pipeline_statistics] {
         DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)};
         uses_push_descriptor = builder.CanUsePushDescriptor();
         descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor);
@@ -250,6 +251,9 @@ GraphicsPipeline::GraphicsPipeline(
         const VkRenderPass render_pass{render_pass_cache.Get(MakeRenderPassKey(key.state))};
         Validate();
         MakePipeline(render_pass);
+        if (pipeline_statistics) {
+            pipeline_statistics->Collect(*pipeline);
+        }
 
         std::lock_guard lock{build_mutex};
         is_built = true;
@@ -782,10 +786,14 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
         }
         */
     }
+    VkPipelineCreateFlags flags{};
+    if (device.IsKhrPipelineEexecutablePropertiesEnabled()) {
+        flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR;
+    }
     pipeline = device.GetLogical().CreateGraphicsPipeline({
         .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
         .pNext = nullptr,
-        .flags = 0,
+        .flags = flags,
         .stageCount = static_cast<u32>(shader_stages.size()),
         .pStages = shader_stages.data(),
         .pVertexInputState = &vertex_input_ci,
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
index 2bd48d697..1c780e944 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
@@ -60,6 +60,7 @@ struct hash<Vulkan::GraphicsPipelineCacheKey> {
 namespace Vulkan {
 
 class Device;
+class PipelineStatistics;
 class RenderPassCache;
 class VKScheduler;
 class VKUpdateDescriptorQueue;
@@ -73,8 +74,9 @@ public:
         VKScheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache,
         VideoCore::ShaderNotify* shader_notify, const Device& device,
         DescriptorPool& descriptor_pool, VKUpdateDescriptorQueue& update_descriptor_queue,
-        Common::ThreadWorker* worker_thread, RenderPassCache& render_pass_cache,
-        const GraphicsPipelineCacheKey& key, std::array<vk::ShaderModule, NUM_STAGES> stages,
+        Common::ThreadWorker* worker_thread, PipelineStatistics* pipeline_statistics,
+        RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key,
+        std::array<vk::ShaderModule, NUM_STAGES> stages,
         const std::array<const Shader::Info*, NUM_STAGES>& infos);
 
     GraphicsPipeline& operator=(GraphicsPipeline&&) noexcept = delete;
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index 57b163247..a37ca1fdf 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -29,6 +29,7 @@
 #include "video_core/renderer_vulkan/fixed_pipeline_state.h"
 #include "video_core/renderer_vulkan/maxwell_to_vk.h"
 #include "video_core/renderer_vulkan/pipeline_helper.h"
+#include "video_core/renderer_vulkan/pipeline_statistics.h"
 #include "video_core/renderer_vulkan/vk_compute_pipeline.h"
 #include "video_core/renderer_vulkan/vk_descriptor_pool.h"
 #include "video_core/renderer_vulkan/vk_pipeline_cache.h"
@@ -389,15 +390,19 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
         size_t total{};
         size_t built{};
         bool has_loaded{};
+        std::unique_ptr<PipelineStatistics> statistics;
     } state;
 
+    if (device.IsKhrPipelineEexecutablePropertiesEnabled()) {
+        state.statistics = std::make_unique<PipelineStatistics>(device);
+    }
     const auto load_compute{[&](std::ifstream& file, FileEnvironment env) {
         ComputePipelineCacheKey key;
         file.read(reinterpret_cast<char*>(&key), sizeof(key));
 
         workers.QueueWork([this, key, env = std::move(env), &state, &callback]() mutable {
             ShaderPools pools;
-            auto pipeline{CreateComputePipeline(pools, key, env, false)};
+            auto pipeline{CreateComputePipeline(pools, key, env, state.statistics.get(), false)};
             std::lock_guard lock{state.mutex};
             if (pipeline) {
                 compute_cache.emplace(key, std::move(pipeline));
@@ -425,7 +430,8 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
             for (auto& env : envs) {
                 env_ptrs.push_back(&env);
             }
-            auto pipeline{CreateGraphicsPipeline(pools, key, MakeSpan(env_ptrs), false)};
+            auto pipeline{CreateGraphicsPipeline(pools, key, MakeSpan(env_ptrs),
+                                                 state.statistics.get(), false)};
 
             std::lock_guard lock{state.mutex};
             graphics_cache.emplace(key, std::move(pipeline));
@@ -445,6 +451,10 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
     lock.unlock();
 
     workers.WaitForRequests();
+
+    if (state.statistics) {
+        state.statistics->Report();
+    }
 }
 
 GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
@@ -486,7 +496,8 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const
 
 std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
     ShaderPools& pools, const GraphicsPipelineCacheKey& key,
-    std::span<Shader::Environment* const> envs, bool build_in_parallel) try {
+    std::span<Shader::Environment* const> envs, PipelineStatistics* statistics,
+    bool build_in_parallel) try {
     LOG_INFO(Render_Vulkan, "0x{:016x}", key.Hash());
     size_t env_index{0};
     std::array<Shader::IR::Program, Maxwell::MaxShaderProgram> programs;
@@ -540,7 +551,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
     Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr};
     return std::make_unique<GraphicsPipeline>(
         maxwell3d, gpu_memory, scheduler, buffer_cache, texture_cache, &shader_notify, device,
-        descriptor_pool, update_descriptor_queue, thread_worker, render_pass_cache, key,
+        descriptor_pool, update_descriptor_queue, thread_worker, statistics, render_pass_cache, key,
         std::move(modules), infos);
 
 } catch (const Shader::Exception& exception) {
@@ -553,7 +564,8 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
     GetGraphicsEnvironments(environments, graphics_key.unique_hashes);
 
     main_pools.ReleaseContents();
-    auto pipeline{CreateGraphicsPipeline(main_pools, graphics_key, environments.Span(), true)};
+    auto pipeline{
+        CreateGraphicsPipeline(main_pools, graphics_key, environments.Span(), nullptr, true)};
     if (!pipeline || pipeline_cache_filename.empty()) {
         return pipeline;
     }
@@ -578,7 +590,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
     env.SetCachedSize(shader->size_bytes);
 
     main_pools.ReleaseContents();
-    auto pipeline{CreateComputePipeline(main_pools, key, env, true)};
+    auto pipeline{CreateComputePipeline(main_pools, key, env, nullptr, true)};
     if (!pipeline || pipeline_cache_filename.empty()) {
         return pipeline;
     }
@@ -591,7 +603,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
 
 std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
     ShaderPools& pools, const ComputePipelineCacheKey& key, Shader::Environment& env,
-    bool build_in_parallel) try {
+    PipelineStatistics* statistics, bool build_in_parallel) try {
     LOG_INFO(Render_Vulkan, "0x{:016x}", key.Hash());
 
     Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()};
@@ -605,8 +617,8 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
     }
     Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr};
     return std::make_unique<ComputePipeline>(device, descriptor_pool, update_descriptor_queue,
-                                             thread_worker, &shader_notify, program.info,
-                                             std::move(spv_module));
+                                             thread_worker, statistics, &shader_notify,
+                                             program.info, std::move(spv_module));
 
 } catch (const Shader::Exception& exception) {
     LOG_ERROR(Render_Vulkan, "{}", exception.what());
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
index efe5a7ed8..4c135b5dd 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
@@ -80,8 +80,9 @@ struct hash<Vulkan::ComputePipelineCacheKey> {
 namespace Vulkan {
 
 class ComputePipeline;
-class Device;
 class DescriptorPool;
+class Device;
+class PipelineStatistics;
 class RasterizerVulkan;
 class RenderPassCache;
 class VKScheduler;
@@ -128,7 +129,8 @@ private:
 
     std::unique_ptr<GraphicsPipeline> CreateGraphicsPipeline(
         ShaderPools& pools, const GraphicsPipelineCacheKey& key,
-        std::span<Shader::Environment* const> envs, bool build_in_parallel);
+        std::span<Shader::Environment* const> envs, PipelineStatistics* statistics,
+        bool build_in_parallel);
 
     std::unique_ptr<ComputePipeline> CreateComputePipeline(const ComputePipelineCacheKey& key,
                                                            const ShaderInfo* shader);
@@ -136,6 +138,7 @@ private:
     std::unique_ptr<ComputePipeline> CreateComputePipeline(ShaderPools& pools,
                                                            const ComputePipelineCacheKey& key,
                                                            Shader::Environment& env,
+                                                           PipelineStatistics* statistics,
                                                            bool build_in_parallel);
 
     const Device& device;