Merge pull request #11475 from GPUCode/renderdoc-hotkey

debug: Add renderdoc capture hotkey

10 files changed, 165 insertions, 48 deletions

diff --git a/src/common/settings.h b/src/common/settings.h
index b15213bd7..82ec9077e 100644
--- a/src/common/settings.h
+++ b/src/common/settings.h
@@ -348,6 +348,8 @@ struct Values {
                                           Category::RendererDebug};
     Setting<bool> disable_shader_loop_safety_checks{
         linkage, false, "disable_shader_loop_safety_checks", Category::RendererDebug};
+    Setting<bool> enable_renderdoc_hotkey{linkage, false, "renderdoc_hotkey",
+                                          Category::RendererDebug};
 
     // System
     SwitchableSetting<Language, true> language_index{linkage,
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 6cd1a28f2..30d2f7df6 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -864,6 +864,8 @@ add_library(core STATIC
     telemetry_session.h
     tools/freezer.cpp
     tools/freezer.h
+    tools/renderdoc.cpp
+    tools/renderdoc.h
 )
 
 if (MSVC)
@@ -879,6 +881,7 @@ else()
         -Werror=conversion
 
         -Wno-sign-conversion
+        -Wno-cast-function-type
 
         $<$<CXX_COMPILER_ID:Clang>:-fsized-deallocation>
     )
@@ -887,7 +890,7 @@ endif()
 create_target_directory_groups(core)
 
 target_link_libraries(core PUBLIC common PRIVATE audio_core network video_core nx_tzdb)
-target_link_libraries(core PUBLIC Boost::headers PRIVATE fmt::fmt nlohmann_json::nlohmann_json mbedtls Opus::opus)
+target_link_libraries(core PUBLIC Boost::headers PRIVATE fmt::fmt nlohmann_json::nlohmann_json mbedtls Opus::opus renderdoc)
 if (MINGW)
     target_link_libraries(core PRIVATE ${MSWSOCK_LIBRARY})
 endif()
diff --git a/src/core/core.cpp b/src/core/core.cpp
index 2d6e61398..e8300cd05 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -51,6 +51,7 @@
 #include "core/reporter.h"
 #include "core/telemetry_session.h"
 #include "core/tools/freezer.h"
+#include "core/tools/renderdoc.h"
 #include "network/network.h"
 #include "video_core/host1x/host1x.h"
 #include "video_core/renderer_base.h"
@@ -281,6 +282,10 @@ struct System::Impl {
         microprofile_cpu[2] = MICROPROFILE_TOKEN(ARM_CPU2);
         microprofile_cpu[3] = MICROPROFILE_TOKEN(ARM_CPU3);
 
+        if (Settings::values.enable_renderdoc_hotkey) {
+            renderdoc_api = std::make_unique<Tools::RenderdocAPI>();
+        }
+
         LOG_DEBUG(Core, "Initialized OK");
 
         return SystemResultStatus::Success;
@@ -521,6 +526,8 @@ struct System::Impl {
     std::unique_ptr<Tools::Freezer> memory_freezer;
     std::array<u8, 0x20> build_id{};
 
+    std::unique_ptr<Tools::RenderdocAPI> renderdoc_api;
+
     /// Frontend applets
     Service::AM::Applets::AppletManager applet_manager;
 
@@ -1024,6 +1031,10 @@ const Network::RoomNetwork& System::GetRoomNetwork() const {
     return impl->room_network;
 }
 
+Tools::RenderdocAPI& System::GetRenderdocAPI() {
+    return *impl->renderdoc_api;
+}
+
 void System::RunServer(std::unique_ptr<Service::ServerManager>&& server_manager) {
     return impl->kernel.RunServer(std::move(server_manager));
 }
diff --git a/src/core/core.h b/src/core/core.h
index fba312125..df20f26f3 100644
--- a/src/core/core.h
+++ b/src/core/core.h
@@ -102,6 +102,10 @@ namespace Network {
 class RoomNetwork;
 }
 
+namespace Tools {
+class RenderdocAPI;
+}
+
 namespace Core {
 
 class ARM_Interface;
@@ -413,6 +417,8 @@ public:
     /// Gets an immutable reference to the Room Network.
     [[nodiscard]] const Network::RoomNetwork& GetRoomNetwork() const;
 
+    [[nodiscard]] Tools::RenderdocAPI& GetRenderdocAPI();
+
     void SetExitLocked(bool locked);
     bool GetExitLocked() const;
 
diff --git a/src/core/tools/renderdoc.cpp b/src/core/tools/renderdoc.cpp
new file mode 100644
index 000000000..44d24822a
--- /dev/null
+++ b/src/core/tools/renderdoc.cpp
@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <renderdoc_app.h>
+
+#include "common/assert.h"
+#include "common/dynamic_library.h"
+#include "core/tools/renderdoc.h"
+
+#ifdef WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+namespace Tools {
+
+RenderdocAPI::RenderdocAPI() {
+#ifdef WIN32
+    if (HMODULE mod = GetModuleHandleA("renderdoc.dll")) {
+        const auto RENDERDOC_GetAPI =
+            reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(mod, "RENDERDOC_GetAPI"));
+        const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdoc_api);
+        ASSERT(ret == 1);
+    }
+#else
+#ifdef ANDROID
+    static constexpr const char RENDERDOC_LIB[] = "libVkLayer_GLES_RenderDoc.so";
+#else
+    static constexpr const char RENDERDOC_LIB[] = "librenderdoc.so";
+#endif
+    if (void* mod = dlopen(RENDERDOC_LIB, RTLD_NOW | RTLD_NOLOAD)) {
+        const auto RENDERDOC_GetAPI =
+            reinterpret_cast<pRENDERDOC_GetAPI>(dlsym(mod, "RENDERDOC_GetAPI"));
+        const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdoc_api);
+        ASSERT(ret == 1);
+    }
+#endif
+}
+
+RenderdocAPI::~RenderdocAPI() = default;
+
+void RenderdocAPI::ToggleCapture() {
+    if (!rdoc_api) [[unlikely]] {
+        return;
+    }
+    if (!is_capturing) {
+        rdoc_api->StartFrameCapture(NULL, NULL);
+    } else {
+        rdoc_api->EndFrameCapture(NULL, NULL);
+    }
+    is_capturing = !is_capturing;
+}
+
+} // namespace Tools
diff --git a/src/core/tools/renderdoc.h b/src/core/tools/renderdoc.h
new file mode 100644
index 000000000..0e5e43da5
--- /dev/null
+++ b/src/core/tools/renderdoc.h
@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+struct RENDERDOC_API_1_6_0;
+
+namespace Tools {
+
+class RenderdocAPI {
+public:
+    explicit RenderdocAPI();
+    ~RenderdocAPI();
+
+    void ToggleCapture();
+
+private:
+    RENDERDOC_API_1_6_0* rdoc_api{};
+    bool is_capturing{false};
+};
+
+} // namespace Tools
diff --git a/src/yuzu/configuration/configure_debug.cpp b/src/yuzu/configuration/configure_debug.cpp
index cbeb8f168..b22fda746 100644
--- a/src/yuzu/configuration/configure_debug.cpp
+++ b/src/yuzu/configuration/configure_debug.cpp
@@ -59,6 +59,8 @@ void ConfigureDebug::SetConfiguration() {
     ui->use_debug_asserts->setChecked(Settings::values.use_debug_asserts.GetValue());
     ui->use_auto_stub->setChecked(Settings::values.use_auto_stub.GetValue());
     ui->enable_all_controllers->setChecked(Settings::values.enable_all_controllers.GetValue());
+    ui->enable_renderdoc_hotkey->setEnabled(runtime_lock);
+    ui->enable_renderdoc_hotkey->setChecked(Settings::values.enable_renderdoc_hotkey.GetValue());
     ui->enable_graphics_debugging->setEnabled(runtime_lock);
     ui->enable_graphics_debugging->setChecked(Settings::values.renderer_debug.GetValue());
     ui->enable_shader_feedback->setEnabled(runtime_lock);
@@ -111,6 +113,7 @@ void ConfigureDebug::ApplyConfiguration() {
     Settings::values.use_auto_stub = ui->use_auto_stub->isChecked();
     Settings::values.enable_all_controllers = ui->enable_all_controllers->isChecked();
     Settings::values.renderer_debug = ui->enable_graphics_debugging->isChecked();
+    Settings::values.enable_renderdoc_hotkey = ui->enable_renderdoc_hotkey->isChecked();
     Settings::values.renderer_shader_feedback = ui->enable_shader_feedback->isChecked();
     Settings::values.cpu_debug_mode = ui->enable_cpu_debugging->isChecked();
     Settings::values.enable_nsight_aftermath = ui->enable_nsight_aftermath->isChecked();
diff --git a/src/yuzu/configuration/configure_debug.ui b/src/yuzu/configuration/configure_debug.ui
index 97c7d9022..66b8b7459 100644
--- a/src/yuzu/configuration/configure_debug.ui
+++ b/src/yuzu/configuration/configure_debug.ui
@@ -18,8 +18,8 @@
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>829</width>
-     <height>758</height>
+     <width>842</width>
+     <height>741</height>
     </rect>
    </property>
    <layout class="QVBoxLayout" name="verticalLayout_1">
@@ -260,7 +260,7 @@
          <string>Graphics</string>
         </property>
         <layout class="QGridLayout" name="gridLayout_2">
-         <item row="3" column="0">
+         <item row="4" column="0">
           <widget class="QCheckBox" name="disable_loop_safety_checks">
            <property name="toolTip">
             <string>When checked, it executes shaders without loop logic changes</string>
@@ -270,33 +270,53 @@
            </property>
           </widget>
          </item>
-         <item row="4" column="0">
-          <widget class="QCheckBox" name="dump_shaders">
+         <item row="8" column="0">
+          <widget class="QCheckBox" name="disable_macro_hle">
            <property name="enabled">
             <bool>true</bool>
            </property>
            <property name="toolTip">
-            <string>When checked, it will dump all the original assembler shaders from the disk shader cache or game as found</string>
+            <string>When checked, it disables the macro HLE functions. Enabling this makes games run slower</string>
            </property>
            <property name="text">
-            <string>Dump Game Shaders</string>
+            <string>Disable Macro HLE</string>
            </property>
           </widget>
          </item>
          <item row="7" column="0">
-          <widget class="QCheckBox" name="disable_macro_hle">
+          <widget class="QCheckBox" name="dump_macros">
            <property name="enabled">
             <bool>true</bool>
            </property>
            <property name="toolTip">
-            <string>When checked, it disables the macro HLE functions. Enabling this makes games run slower</string>
+            <string>When checked, it will dump all the macro programs of the GPU</string>
            </property>
            <property name="text">
-            <string>Disable Macro HLE</string>
+            <string>Dump Maxwell Macros</string>
            </property>
           </widget>
          </item>
-         <item row="5" column="0">
+         <item row="3" column="0">
+          <widget class="QCheckBox" name="enable_nsight_aftermath">
+           <property name="toolTip">
+            <string>When checked, it enables Nsight Aftermath crash dumps</string>
+           </property>
+           <property name="text">
+            <string>Enable Nsight Aftermath</string>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="0">
+          <widget class="QCheckBox" name="enable_shader_feedback">
+           <property name="toolTip">
+            <string>When checked, yuzu will log statistics about the compiled pipeline cache</string>
+           </property>
+           <property name="text">
+            <string>Enable Shader Feedback</string>
+           </property>
+          </widget>
+         </item>
+         <item row="6" column="0">
           <widget class="QCheckBox" name="disable_macro_jit">
            <property name="enabled">
             <bool>true</bool>
@@ -309,6 +329,22 @@
            </property>
           </widget>
          </item>
+         <item row="9" column="0">
+          <spacer name="verticalSpacer_5">
+           <property name="orientation">
+            <enum>Qt::Vertical</enum>
+           </property>
+           <property name="sizeType">
+            <enum>QSizePolicy::Preferred</enum>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>20</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
          <item row="0" column="0">
           <widget class="QCheckBox" name="enable_graphics_debugging">
            <property name="enabled">
@@ -322,55 +358,26 @@
            </property>
           </widget>
          </item>
-         <item row="6" column="0">
-          <widget class="QCheckBox" name="dump_macros">
+         <item row="5" column="0">
+          <widget class="QCheckBox" name="dump_shaders">
            <property name="enabled">
             <bool>true</bool>
            </property>
            <property name="toolTip">
-            <string>When checked, it will dump all the macro programs of the GPU</string>
+            <string>When checked, it will dump all the original assembler shaders from the disk shader cache or game as found</string>
            </property>
            <property name="text">
-            <string>Dump Maxwell Macros</string>
+            <string>Dump Game Shaders</string>
            </property>
           </widget>
          </item>
          <item row="1" column="0">
-          <widget class="QCheckBox" name="enable_shader_feedback">
-           <property name="toolTip">
-            <string>When checked, yuzu will log statistics about the compiled pipeline cache</string>
-           </property>
-           <property name="text">
-            <string>Enable Shader Feedback</string>
-           </property>
-          </widget>
-         </item>
-         <item row="2" column="0">
-          <widget class="QCheckBox" name="enable_nsight_aftermath">
-           <property name="toolTip">
-            <string>When checked, it enables Nsight Aftermath crash dumps</string>
-           </property>
+          <widget class="QCheckBox" name="enable_renderdoc_hotkey">
            <property name="text">
-            <string>Enable Nsight Aftermath</string>
+            <string>Enable Renderdoc Hotkey</string>
            </property>
           </widget>
          </item>
-         <item row="8" column="0">
-          <spacer name="verticalSpacer_5">
-           <property name="orientation">
-            <enum>Qt::Vertical</enum>
-           </property>
-           <property name="sizeType">
-            <enum>QSizePolicy::Preferred</enum>
-           </property>
-           <property name="sizeHint" stdset="0">
-            <size>
-             <width>20</width>
-             <height>0</height>
-            </size>
-           </property>
-          </spacer>
-         </item>
         </layout>
        </widget>
       </item>
diff --git a/src/yuzu/hotkeys.h b/src/yuzu/hotkeys.h
index 848239c35..56eee8d82 100644
--- a/src/yuzu/hotkeys.h
+++ b/src/yuzu/hotkeys.h
@@ -4,10 +4,12 @@
 #pragma once
 
 #include <map>
+#include <QKeySequence>
+#include <QString>
+#include <QWidget>
 #include "core/hid/hid_types.h"
 
 class QDialog;
-class QKeySequence;
 class QSettings;
 class QShortcut;
 class ControllerShortcut;
diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp
index 97d216638..d32aa9615 100644
--- a/src/yuzu/main.cpp
+++ b/src/yuzu/main.cpp
@@ -9,6 +9,7 @@
 #include <memory>
 #include <thread>
 #include "core/loader/nca.h"
+#include "core/tools/renderdoc.h"
 #ifdef __APPLE__
 #include <unistd.h> // for chdir
 #endif
@@ -1348,6 +1349,11 @@ void GMainWindow::InitializeHotkeys() {
     connect_shortcut(QStringLiteral("Toggle Framerate Limit"), [] {
         Settings::values.use_speed_limit.SetValue(!Settings::values.use_speed_limit.GetValue());
     });
+    connect_shortcut(QStringLiteral("Toggle Renderdoc Capture"), [this] {
+        if (Settings::values.enable_renderdoc_hotkey) {
+            system->GetRenderdocAPI().ToggleCapture();
+        }
+    });
     connect_shortcut(QStringLiteral("Toggle Mouse Panning"), [&] {
         if (Settings::values.mouse_enabled) {
             Settings::values.mouse_panning = false;